Schedule data export tasks in sidekiq-cron

- Split combined XML and CSV data jobs into seperate background jobs
- Add sidekiq-cron to the project
- Schedule new XML and CSV jobs to run every day at 5am
- Adjust data export Rake tasks to use new job

Gemfile

@@ -50,7 +50,7 @@ gem "paper_trail"
 gem "paper_trail-globalid"
 # Request rate limiting
 gem "rack-attack"
-gem "redis"
+gem "redis", "~> 4.8"
 # Receive exceptions and configure alerts
 gem "sentry-rails"
 gem "sentry-ruby"
@@ -60,6 +60,7 @@ gem "possessive"
 gem "auto_strip_attributes"
 # Use sidekiq for background processing
 gem "sidekiq"
+gem "sidekiq-cron"
 
 group :development, :test do
   # Check gems for known vulnerabilities

Gemfile.lock

@@ -157,7 +157,13 @@ GEM
       rainbow
       rubocop
       smart_properties
+<<<<<<< HEAD
     erubi (1.12.0)
+=======
+    erubi (1.11.0)
+    et-orbi (1.2.7)
+      tzinfo
+>>>>>>> 6991b683 (Schedule data export tasks in sidekiq-cron)
     excon (0.92.5)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
@@ -167,7 +173,14 @@ GEM
     faker (2.23.0)
       i18n (>= 1.8.11, < 2)
     ffi (1.15.5)
+<<<<<<< HEAD
     globalid (1.0.1)
+=======
+    fugit (1.7.1)
+      et-orbi (~> 1, >= 1.2.7)
+      raabro (~> 1.4)
+    globalid (1.0.0)
+>>>>>>> 6991b683 (Schedule data export tasks in sidekiq-cron)
       activesupport (>= 5.0)
     govuk-components (3.2.1)
       actionpack (>= 6.1)
@@ -267,8 +280,14 @@ GEM
     public_suffix (5.0.0)
     puma (5.6.5)
       nio4r (~> 2.0)
+<<<<<<< HEAD
     racc (1.6.2)
     rack (2.2.6.2)
+=======
+    raabro (1.4.0)
+    racc (1.6.0)
+    rack (2.2.4)
+>>>>>>> 6991b683 (Schedule data export tasks in sidekiq-cron)
     rack-attack (6.6.1)
       rack (>= 1.0, < 3)
     rack-mini-profiler (2.3.4)
@@ -308,10 +327,7 @@ GEM
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     redcarpet (3.5.1)
-    redis (5.0.5)
+    redis (4.8.0)
-      redis-client (>= 0.9.0)
-    redis-client (0.9.0)
-      connection_pool
     regexp_parser (2.5.0)
     request_store (1.5.1)
       rack (>= 1.4)
@@ -385,6 +401,9 @@ GEM
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.5.0)
+    sidekiq-cron (1.8.0)
+      fugit (~> 1)
+      sidekiq (>= 4.2.1)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -467,7 +486,7 @@ DEPENDENCIES
   rack-attack
   rack-mini-profiler (~> 2.0)
   rails (~> 7.0.2)
-  redis
+  redis (~> 4.8)
   roo
   rspec-rails
   rubocop-govuk (= 4.3.0)
@@ -477,6 +496,7 @@ DEPENDENCIES
   sentry-rails
   sentry-ruby
   sidekiq
+  sidekiq-cron
   simplecov
   stimulus-rails
   timecop (~> 0.9.4)

Procfile.dev

@@ -1,2 +1,3 @@
 web: bin/rails server -p 3000
+redis: redis-server
 js: yarn build --watch

app/jobs/data_export_csv_job.rb

@@ -0,0 +1,10 @@
+class DataExportCsvJob < ApplicationJob
+  queue_as :default
+
+  def perform
+    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
+    export_service = Exports::LettingsLogExportService.new(storage_service)
+
+    export_service.export_csv_lettings_logs
+  end
+end

app/jobs/data_export_xml_job.rb

@@ -0,0 +1,10 @@
+class DataExportXmlJob < ApplicationJob
+  queue_as :default
+
+  def perform(full_update: false)
+    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
+    export_service = Exports::LettingsLogExportService.new(storage_service)
+
+    export_service.export_xml_lettings_logs(full_update:)
+  end
+end

config/initializers/sidekiq.rb

@@ -1,4 +1,5 @@
 require "sidekiq/web"
+require "sidekiq/cron/web"
 
 if Rails.env.staging? || Rails.env.production?
   redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"]
@@ -23,3 +24,12 @@ if Rails.env.review?
     config.redis = { url: redis_url }
   end
 end
+
+# Until https://github.com/sidekiq-cron/sidekiq-cron/issues/357 is fixed.
+Redis.silence_deprecations = true
+
+Sidekiq.configure_server do |config|
+  config.on(:startup) do
+    Sidekiq::Cron::Job.load_from_hash YAML.load_file("config/sidekiq_cron_schedule.yml")
+  end
+end

config/sidekiq_cron_schedule.yml

@@ -0,0 +1,8 @@
+data_export_csv:
+  cron: "every day at 5am"
+  class: "DataExportCsvJob"
+  queue: default
+data_export_xml:
+  cron: "every day at 5am"
+  class: "DataExportXmlJob"
+  queue: default

lib/tasks/data_export.rake

@@ -1,17 +1,14 @@
 namespace :core do
+  desc "Export data CSVs for import into Central Data System (CDS)"
+  task data_export_csv: :environment do |_task, _args|
+    DataExportCsvJob.perform_later
+  end
+
   desc "Export data XMLs for import into Central Data System (CDS)"
-  task :data_export, %i[format full_update] => :environment do |_task, args|
+  task :data_export_xml, %i[full_update] => :environment do |_task, args|
-    format = args[:format]
     full_update = args[:full_update].present? && args[:full_update] == "true"
 
-    storage_service = Storage::S3Service.new(Configuration::PaasConfigurationService.new, ENV["EXPORT_PAAS_INSTANCE"])
+    DataExportXmlJob.perform_later(full_update:)
-    export_service = Exports::LettingsLogExportService.new(storage_service)
-
-    if format.present? && format == "CSV"
-      export_service.export_csv_lettings_logs
-    else
-      export_service.export_xml_lettings_logs(full_update:)
-    end
   end
 end
 

obj_to_form.rb

@@ -0,0 +1,11 @@
+def filename(klass) = [klass.to_s.underscore, "rb"].join(".")
+
+def write(path, body)
+  path = Rails.root.join("app", "models", path)
+  puts "#{path}"
+  File.write(path, body) unless File.exist?(path)
+end
+FormHandler.instance.current_lettings_form.sections.each do |section|
+  path = filename(section.class)
+  write(path, "hi")
+end; nil

spec/lib/tasks/data_export_spec.rb

@@ -2,8 +2,6 @@ require "rails_helper"
 require "rake"
 
 describe "rake core:data_export", type: task do
-  subject(:task) { Rake::Task["core:data_export"] }
-
   let(:paas_instance) { "paas_export_instance" }
   let(:storage_service) { instance_double(Storage::S3Service) }
   let(:paas_config_service) { instance_double(Configuration::PaasConfigurationService) }
@@ -22,22 +20,18 @@ describe "rake core:data_export", type: task do
   end
 
   context "when exporting lettings logs with no parameters" do
-    it "starts the XML export process" do
+    let(:task) { Rake::Task["core:data_export_xml"] }
-      expect(Storage::S3Service).to receive(:new).with(paas_config_service, paas_instance)
-      expect(Exports::LettingsLogExportService).to receive(:new).with(storage_service)
-      expect(export_service).to receive(:export_xml_lettings_logs)
 
-      task.invoke
+    it "starts the XML export process" do
+      expect { task.invoke }.to enqueue_job(DataExportXmlJob)
     end
   end
 
   context "when exporting lettings logs with CSV format" do
-    it "starts the CSV export process" do
+    let(:task) { Rake::Task["core:data_export_csv"] }
-      expect(Storage::S3Service).to receive(:new).with(paas_config_service, paas_instance)
-      expect(Exports::LettingsLogExportService).to receive(:new).with(storage_service)
-      expect(export_service).to receive(:export_csv_lettings_logs)
 
-      task.invoke("CSV", "false")
+    it "starts the CSV export process" do
+      expect { task.invoke }.to enqueue_job(DataExportCsvJob)
     end
   end
 end