diff --git a/Gemfile b/Gemfile index 599c5d369..3c8d77954 100644 --- a/Gemfile +++ b/Gemfile @@ -29,6 +29,8 @@ gem "discard" gem "activeadmin" # Admin charts gem "chartkick" +# Spreadsheet parsing +gem "roo" # Json Schema gem "json-schema" gem "uk_postcode" diff --git a/Gemfile.lock b/Gemfile.lock index d2a3b4a15..04db4e6f3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -294,6 +294,9 @@ GEM actionpack (>= 5.0) railties (>= 5.0) rexml (3.2.5) + roo (2.8.3) + nokogiri (~> 1) + rubyzip (>= 1.3.0, < 3.0.0) rubocop (1.21.0) parallel (~> 1.10) parser (>= 3.0.0.0) @@ -409,6 +412,7 @@ DEPENDENCIES puma (~> 5.0) rack-mini-profiler (~> 2.0) rails (~> 6.1.4) + roo rspec-core! rspec-expectations! rspec-mocks! diff --git a/app/controllers/bulk_upload_controller.rb b/app/controllers/bulk_upload_controller.rb new file mode 100644 index 000000000..ba552cb49 --- /dev/null +++ b/app/controllers/bulk_upload_controller.rb @@ -0,0 +1,22 @@ +class BulkUploadController < ApplicationController + def show + @bulk_upload = BulkUpload.new(nil, nil) + render "case_logs/bulk_upload" + end + + def bulk_upload + file = upload_params.tempfile + content_type = upload_params.content_type + @bulk_upload = BulkUpload.new(file, content_type) + @bulk_upload.process + if @bulk_upload.errors.present? + render "case_logs/bulk_upload", status: :unprocessable_entity + else + redirect_to(case_logs_path) + end + end + + def upload_params + params.require("bulk_upload")["case_log_bulk_upload"] + end +end diff --git a/app/models/bulk_upload.rb b/app/models/bulk_upload.rb new file mode 100644 index 000000000..f546ea56d --- /dev/null +++ b/app/models/bulk_upload.rb @@ -0,0 +1,199 @@ +class BulkUpload + include ActiveModel::Model + include ActiveModel::Validations + include ActiveModel::Conversion + + SPREADSHEET_CONTENT_TYPES = %w[ + application/vnd.ms-excel + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + ].freeze + + FIRST_DATA_ROW = 7 + + def initialize(file, content_type) + @file = file + @content_type = content_type + end + + def process + return unless valid_content_type? + + xlsx = Roo::Spreadsheet.open(@file, extension: :xlsx) + sheet = xlsx.sheet(0) + last_row = sheet.last_row + if last_row < FIRST_DATA_ROW + errors.add(:case_log_bulk_upload, "No data found") + else + data_range = FIRST_DATA_ROW..last_row + data_range.map do |row_num| + case_log = CaseLog.create + map_row(sheet.row(row_num)).each do |attr_key, attr_val| + begin + case_log.update_attribute(attr_key, attr_val) + rescue ArgumentError + end + end + end + end + end + + def valid_content_type? + if SPREADSHEET_CONTENT_TYPES.include?(@content_type) + true + else + errors.add(:case_log_bulk_upload, "Invalid file type") + false + end + end + + def map_row(row) + { + lettype: row[1], + landlord: row[2], + # reg_num_la_core_code: row[3], + # managementgroup: row[4], + # schemecode: row[5], + # firstletting: row[6], + tenant_code: row[7], + startertenancy: row[8], + tenancy: row[9], + tenancyother: row[10], + # tenancyduration: row[11], + other_hhmemb: other_hhmemb(row), + hhmemb: other_hhmemb(row) + 1, + age1: row[12], + age2: row[13], + age3: row[14], + age4: row[15], + age5: row[16], + age6: row[17], + age7: row[18], + age8: row[19], + sex1: row[20], + sex2: row[21], + sex3: row[22], + sex4: row[23], + sex5: row[24], + sex6: row[25], + sex7: row[26], + sex8: row[27], + relat2: row[28], + relat3: row[29], + relat4: row[30], + relat5: row[31], + relat6: row[32], + relat7: row[33], + relat8: row[34], + ecstat1: row[35], + ecstat2: row[36], + ecstat3: row[37], + ecstat4: row[38], + ecstat5: row[39], + ecstat6: row[40], + ecstat7: row[41], + ecstat8: row[42], + ethnic: row[43], + national: row[44], + armed_forces: row[45], + reservist: row[46], + preg_occ: row[47], + hb: row[48], + benefits: row[49], + net_income_known: row[50].present? ? 1 : nil, + earnings: row[50], + # increfused: row[51], + reason: row[52], + other_reason_for_leaving_last_settled_home: row[53], + underoccupation_benefitcap: row[54], + housingneeds_a: row[55], + housingneeds_b: row[56], + housingneeds_c: row[57], + housingneeds_f: row[58], + housingneeds_g: row[59], + housingneeds_h: row[60], + prevten: row[61], + prevloc: row[62], + # ppostc1: row[63], + # ppostc2: row[64], + # prevpco_unknown: row[65], + layear: row[66], + lawaitlist: row[67], + homeless: row[68], + reasonpref: row[69], + rp_homeless: row[70], + rp_insan_unsat: row[71], + rp_medwel: row[72], + rp_hardship: row[73], + rp_dontknow: row[74], + cbl: row[75], + chr: row[76], + cap: row[77], + # referral_source: row[78], + period: row[79], + brent: row[80], + scharge: row[81], + pscharge: row[82], + supcharg: row[83], + tcharge: row[84], + # tcharge_care_homes: row[85], + # no_rent_or_charge: row[86], + hbrentshortfall: row[87], + tshortfall: row[88], + property_void_date: row[89].to_s + row[90].to_s + row[91].to_s, + # property_void_date_day: row[89], + # property_void_date_month: row[90], + # property_void_date_year: row[91], + majorrepairs: row[92].present? ? "1" : nil, + mrcdate: row[92].to_s + row[93].to_s + row[94].to_s, + mrcday: row[92], + mrcmonth: row[93], + mrcyear: row[94], + # supported_scheme: row[95], + startdate: row[96].to_s + row[97].to_s + row[98].to_s, + # startdate_day: row[96], + # startdate_month: row[97], + # startdate_year: row[98], + offered: row[99], + # property_reference: row[100], + beds: row[101], + unittype_gn: row[102], + property_building_type: row[103], + wchair: row[104], + property_relet: row[105], + rsnvac: row[106], + la: row[107], + # postcode: row[108], + # postcod2: row[109], + # row[110] removed + property_owner_organisation: row[111], + # username: row[112], + property_manager_organisation: row[113], + leftreg: row[114], + # uprn: row[115], + incfreq: row[116], + # sheltered_accom: row[117], + illness: row[118], + illness_type_1: row[119], + illness_type_2: row[120], + illness_type_3: row[121], + illness_type_4: row[122], + illness_type_8: row[123], + illness_type_5: row[124], + illness_type_6: row[125], + illness_type_7: row[126], + illness_type_9: row[127], + illness_type_10: row[128], + # london_affordable: row[129], + rent_type: row[130], + intermediate_rent_product_name: row[131], + # data_protection: row[132], + sale_or_letting: "letting", + gdpr_acceptance: 1, + gdpr_declined: 0, + } + end + + def other_hhmemb(row) + [13, 14, 15, 16, 17, 18, 19].count { |idx| row[idx].present? } + end +end diff --git a/app/models/case_log.rb b/app/models/case_log.rb index f73df0264..5b5dc6f40 100644 --- a/app/models/case_log.rb +++ b/app/models/case_log.rb @@ -39,7 +39,6 @@ class CaseLog < ApplicationRecord include SoftValidations include DbEnums default_scope -> { kept } - scope :not_completed, -> { where.not(status: "completed") } validates_with CaseLogValidator before_save :update_status! @@ -129,6 +128,8 @@ class CaseLog < ApplicationRecord end def weekly_net_income + return unless earnings && incfreq + case incfreq when "Weekly" earnings diff --git a/app/views/case_logs/bulk_upload.html.erb b/app/views/case_logs/bulk_upload.html.erb new file mode 100644 index 000000000..02cb109df --- /dev/null +++ b/app/views/case_logs/bulk_upload.html.erb @@ -0,0 +1,10 @@ +
+ <%= form_for @bulk_upload, url: bulk_upload_case_logs_path, method: "post", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %> + <%= f.govuk_error_summary %> + <%= f.govuk_file_field :case_log_bulk_upload, + label: { text: "Bulk Upload", size: "l" }, + hint: { text: "Upload a spreadsheet using the template" } + %> + <%= f.govuk_submit "Upload" %> + <% end %> +
diff --git a/app/views/form/page.html.erb b/app/views/form/page.html.erb index d47e89e7b..238f11439 100644 --- a/app/views/form/page.html.erb +++ b/app/views/form/page.html.erb @@ -10,7 +10,7 @@ <%= page_info["header"] %> <% end %> - <%= form_with model: @case_log, method: "submit_form", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %> + <%= form_with model: @case_log, url: form_case_log_path(@case_log), method: "post", builder: GOVUKDesignSystemFormBuilder::FormBuilder do |f| %> <%= f.govuk_error_summary %> <% page_info["questions"].map do |question_key, question| %>
<%= display_question_key_div(page_info, question_key) %> > diff --git a/config/routes.rb b/config/routes.rb index 3bbc0e161..79bbece4e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -4,15 +4,24 @@ Rails.application.routes.draw do root to: "test#index" get "about", to: "about#index" - post "/case_logs/:id", to: "case_logs#submit_form" - form_handler = FormHandler.instance form = form_handler.get_form("2021_2022") + resources :case_logs do + collection do + post "/bulk_upload", to: "bulk_upload#bulk_upload" + get "/bulk_upload", to: "bulk_upload#show" + end + + member do + post "/form", to: "case_logs#submit_form" + end + form.all_pages.keys.map do |page| get page.to_s, to: "case_logs##{page}" - get "#{page}/soft_validations", to: "soft_validations#show" + get "#{page}/soft_validations", to: "soft_validations#show" if form.soft_validations_for_page(page) end + form.all_subsections.keys.map do |subsection| get "#{subsection}/check_answers", to: "case_logs#check_answers" end diff --git a/db/schema.rb b/db/schema.rb index 1ef990508..614c895c2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -122,6 +122,8 @@ ActiveRecord::Schema.define(version: 2021_11_12_105348) do t.integer "rp_dontknow" t.datetime "discarded_at" t.string "tenancyother" + t.integer "override_net_income_validation" + t.string "net_income_known" t.string "gdpr_acceptance" t.string "gdpr_declined" t.string "property_owner_organisation" @@ -133,8 +135,6 @@ ActiveRecord::Schema.define(version: 2021_11_12_105348) do t.string "needs_type" t.string "sale_completion_date" t.string "purchaser_code" - t.integer "override_net_income_validation" - t.string "net_income_known" t.integer "reason" t.string "propcode" t.integer "majorrepairs" diff --git a/spec/fixtures/files/2021_22_lettings_bulk_upload.xlsx b/spec/fixtures/files/2021_22_lettings_bulk_upload.xlsx new file mode 100644 index 000000000..c8e86cdbe Binary files /dev/null and b/spec/fixtures/files/2021_22_lettings_bulk_upload.xlsx differ diff --git a/spec/fixtures/files/2021_22_lettings_bulk_upload_empty.xlsx b/spec/fixtures/files/2021_22_lettings_bulk_upload_empty.xlsx new file mode 100644 index 000000000..f1b7de0bb Binary files /dev/null and b/spec/fixtures/files/2021_22_lettings_bulk_upload_empty.xlsx differ diff --git a/spec/fixtures/files/random.txt b/spec/fixtures/files/random.txt new file mode 100644 index 000000000..e69de29bb diff --git a/spec/requests/bulk_upload_controller_spec.rb b/spec/requests/bulk_upload_controller_spec.rb new file mode 100644 index 000000000..6b9ab16e0 --- /dev/null +++ b/spec/requests/bulk_upload_controller_spec.rb @@ -0,0 +1,60 @@ +require "rails_helper" + +RSpec.describe BulkUploadController, type: :request do + let(:url) { "/case_logs/bulk_upload" } + + describe "GET #show" do + before do + get url, params: {} + end + + it "returns a success response" do + expect(response).to be_successful + end + + it "returns a page with a file upload form" do + expect(response.body).to match(/