Manny Dinssa
2 years ago
committed by
GitHub
62 changed files with 2842 additions and 234 deletions
@ -0,0 +1,10 @@ |
|||||||
|
class CsvVariableDefinition < ApplicationRecord |
||||||
|
validates :variable, presence: true |
||||||
|
validates :definition, presence: true |
||||||
|
validates :log_type, presence: true, inclusion: { in: %w[lettings sales] } |
||||||
|
validates :year, presence: true, numericality: { only_integer: true, greater_than_or_equal_to: 2000, less_than_or_equal_to: 2099 } |
||||||
|
attribute :last_accessed, :datetime |
||||||
|
|
||||||
|
scope :lettings, -> { where(log_type: "lettings") } |
||||||
|
scope :sales, -> { where(log_type: "sales") } |
||||||
|
end |
||||||
@ -0,0 +1,53 @@ |
|||||||
|
require "csv" |
||||||
|
|
||||||
|
module Imports |
||||||
|
class VariableDefinitionsService |
||||||
|
attr_reader :path, :count |
||||||
|
|
||||||
|
def initialize(path:) |
||||||
|
@path = path |
||||||
|
@count = 0 |
||||||
|
end |
||||||
|
|
||||||
|
def call |
||||||
|
files = Dir.glob(File.join(@path, "*.csv")) |
||||||
|
files.each do |file| |
||||||
|
process_file(file) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
private |
||||||
|
|
||||||
|
def process_file(file) |
||||||
|
file_name = File.basename(file, ".csv") |
||||||
|
parsed_file_name = file_name.split("_") |
||||||
|
log_type = parsed_file_name[0] |
||||||
|
year = "20#{parsed_file_name[2]}".to_i |
||||||
|
|
||||||
|
records_added = 0 |
||||||
|
|
||||||
|
CSV.foreach(file) do |row| |
||||||
|
next if row.empty? |
||||||
|
|
||||||
|
variable = row[0].downcase |
||||||
|
definition = row[1..].join(",") |
||||||
|
next if variable.nil? || definition.nil? |
||||||
|
|
||||||
|
existing_record = CsvVariableDefinition.find_by(variable: variable.strip, definition: definition.strip, log_type:) |
||||||
|
|
||||||
|
if existing_record.nil? |
||||||
|
CsvVariableDefinition.create!( |
||||||
|
variable: variable.strip, |
||||||
|
definition: definition.strip, |
||||||
|
log_type:, |
||||||
|
year:, |
||||||
|
) |
||||||
|
records_added += 1 |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
Rails.logger.debug "Added #{records_added} variable/definition records for file: #{file_name}. Duplicates excluded." |
||||||
|
@count += records_added |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 25.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
@ -0,0 +1,15 @@ |
|||||||
|
class CreateCsvVariableDefinitions < ActiveRecord::Migration[7.0] |
||||||
|
def change |
||||||
|
create_table :csv_variable_definitions do |t| |
||||||
|
t.string :variable, null: false |
||||||
|
t.string :definition, null: false |
||||||
|
t.string :log_type, null: false |
||||||
|
t.integer :year, null: false |
||||||
|
t.datetime :last_accessed |
||||||
|
t.timestamps |
||||||
|
end |
||||||
|
|
||||||
|
add_check_constraint :csv_variable_definitions, "log_type IN ('lettings', 'sales')", name: "log_type_check" |
||||||
|
add_check_constraint :csv_variable_definitions, "year BETWEEN 2000 AND 2099", name: "year_check" |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,5 @@ |
|||||||
|
class AddPhoneExtensionToUsers < ActiveRecord::Migration[7.0] |
||||||
|
def change |
||||||
|
add_column :users, :phone_extension, :string |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,4 @@ |
|||||||
|
desc "Clear unconfimed emails for deactivated users" |
||||||
|
task clear_unconfirmed_emails: :environment do |
||||||
|
User.deactivated.where.not(unconfirmed_email: nil).update(unconfirmed_email: nil) |
||||||
|
end |
||||||
@ -0,0 +1,9 @@ |
|||||||
|
namespace :data_import do |
||||||
|
desc "Add CsvVariableDefinition records for each file in the specified directory" |
||||||
|
task :add_variable_definitions, [:path] => :environment do |_task, args| |
||||||
|
path = Rails.root.join(args[:path]) |
||||||
|
service = Imports::VariableDefinitionsService.new(path:) |
||||||
|
service.call |
||||||
|
Rails.logger.info "CSV Variable Definitions: #{service.count} total records added" |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,8 @@ |
|||||||
|
FactoryBot.define do |
||||||
|
factory :csv_variable_definition do |
||||||
|
variable { "variable" } |
||||||
|
definition { "definition" } |
||||||
|
log_type { "lettings" } |
||||||
|
year { 2024 } |
||||||
|
end |
||||||
|
end |
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 25.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
|
Can't render this file because it has a wrong number of fields in line 20.
|
@ -0,0 +1,36 @@ |
|||||||
|
require "rails_helper" |
||||||
|
require "rake" |
||||||
|
|
||||||
|
RSpec.describe "clear_unconfirmed_emails" do |
||||||
|
describe ":clear_unconfirmed_emails", type: :task do |
||||||
|
subject(:task) { Rake::Task["clear_unconfirmed_emails"] } |
||||||
|
|
||||||
|
before do |
||||||
|
Rake.application.rake_require("tasks/clear_unconfirmed_emails") |
||||||
|
Rake::Task.define_task(:environment) |
||||||
|
task.reenable |
||||||
|
end |
||||||
|
|
||||||
|
context "when the rake task is run" do |
||||||
|
context "and there are deactivated users with unconfirmed emails" do |
||||||
|
let!(:user) { create(:user, active: false, unconfirmed_email: "some_email@example.com") } |
||||||
|
|
||||||
|
it "clears unconfirmed_email" do |
||||||
|
task.invoke |
||||||
|
|
||||||
|
expect(user.reload.unconfirmed_email).to eq(nil) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
context "and there are active users with unconfirmed emails" do |
||||||
|
let!(:user) { create(:user, active: true, unconfirmed_email: "some_email@example.com") } |
||||||
|
|
||||||
|
it "does not clear unconfirmed_email" do |
||||||
|
task.invoke |
||||||
|
|
||||||
|
expect(user.reload.unconfirmed_email).not_to eq(nil) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,41 @@ |
|||||||
|
require "rails_helper" |
||||||
|
require "rake" |
||||||
|
|
||||||
|
RSpec.describe "log_variable_definitions" do |
||||||
|
describe ":add_variable_definitions", type: :task do |
||||||
|
subject(:task) { Rake::Task["data_import:add_variable_definitions"] } |
||||||
|
|
||||||
|
let(:path) { "spec/fixtures/variable_definitions" } |
||||||
|
|
||||||
|
before do |
||||||
|
Rake.application.rake_require("tasks/log_variable_definitions") |
||||||
|
Rake::Task.define_task(:environment) |
||||||
|
task.reenable |
||||||
|
end |
||||||
|
|
||||||
|
it "adds CsvVariableDefinition records from each file in the specified directory" do |
||||||
|
expect { task.invoke(path) }.to change(CsvVariableDefinition, :count).by(416) |
||||||
|
end |
||||||
|
|
||||||
|
it "handles an empty directory without errors" do |
||||||
|
empty_path = "spec/fixtures/empty_directory" |
||||||
|
FileUtils.mkdir_p(empty_path) |
||||||
|
expect { task.invoke(empty_path) }.not_to raise_error |
||||||
|
expect(CsvVariableDefinition.count).to eq(0) |
||||||
|
end |
||||||
|
|
||||||
|
it "does not create duplicate records if run multiple times" do |
||||||
|
CsvVariableDefinition.delete_all |
||||||
|
initial_count = CsvVariableDefinition.count |
||||||
|
|
||||||
|
task.invoke(path) |
||||||
|
first_run_count = CsvVariableDefinition.count |
||||||
|
|
||||||
|
task.invoke(path) |
||||||
|
second_run_count = CsvVariableDefinition.count |
||||||
|
|
||||||
|
expect(first_run_count).to eq(initial_count + 416) |
||||||
|
expect(second_run_count).to eq(first_run_count) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
@ -0,0 +1,44 @@ |
|||||||
|
require "rails_helper" |
||||||
|
|
||||||
|
RSpec.describe Imports::VariableDefinitionsService, type: :service do |
||||||
|
let(:path) { "spec/fixtures/variable_definitions" } |
||||||
|
let(:service) { described_class.new(path:) } |
||||||
|
|
||||||
|
describe "#initialize" do |
||||||
|
it "initializes with the correct path and count" do |
||||||
|
expect(service.path).to eq(path) |
||||||
|
expect(service.count).to eq(0) |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
describe "#call" do |
||||||
|
before do |
||||||
|
allow(Dir).to receive(:glob).and_return(%w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv]) |
||||||
|
allow(service).to receive(:process_file) |
||||||
|
end |
||||||
|
|
||||||
|
it "processes each file in the directory" do |
||||||
|
service.call |
||||||
|
%w[lettings_download_23_24.csv lettings_download_24_25.csv sales_download_23_24.csv sales_download_24_25.csv].each do |file| |
||||||
|
expect(service).to have_received(:process_file).with(file) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
|
||||||
|
describe "#process_file" do |
||||||
|
let(:file) { "spec/fixtures/variable_definitions/lettings_download_23_24.csv" } |
||||||
|
let(:csv_content) { [["id", "Log ID"], ["status", "Status of log"], ["duplicate_set_id", "ID of a set of duplicate logs"]] } |
||||||
|
|
||||||
|
before do |
||||||
|
allow(CSV).to receive(:foreach).and_yield(csv_content[0]).and_yield(csv_content[1]).and_yield(csv_content[2]) |
||||||
|
end |
||||||
|
|
||||||
|
context "when no existing record" do |
||||||
|
it "creates new records" do |
||||||
|
expect { |
||||||
|
service.send(:process_file, file) |
||||||
|
}.to change(CsvVariableDefinition, :count).by(3) |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
|
end |
||||||
Loading…
Reference in new issue