diff --git a/.github/workflows/review_pipeline.yml b/.github/workflows/review_pipeline.yml new file mode 100644 index 000000000..8bf7ddb57 --- /dev/null +++ b/.github/workflows/review_pipeline.yml @@ -0,0 +1,158 @@ +name: Review app pipeline + +on: + pull_request: + types: + - opened + - synchronize + - reopened + workflow_dispatch: + +defaults: + run: + shell: bash + +jobs: + postgres: + name: Provision postgres + runs-on: ubuntu-latest + environment: staging + + steps: + - name: Install Cloud Foundry CLI + run: | + wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - + echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list + sudo apt-get update + sudo apt-get install cf8-cli + + - name: Provision postgres + env: + CF_USERNAME: ${{ secrets.CF_USERNAME }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD }} + CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }} + CF_SPACE: dev + CF_ORG: ${{ secrets.CF_ORG }} + run: | + cf api $CF_API_ENDPOINT + cf auth + cf target -o $CF_ORG -s $CF_SPACE + cf create-service postgres tiny-unencrypted-13 dluhc-core-review-${{ github.event.pull_request.number }}-postgres --wait + + redis: + name: Provision redis + runs-on: ubuntu-latest + environment: staging + + steps: + - name: Install Cloud Foundry CLI + run: | + wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - + echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list + sudo apt-get update + sudo apt-get install cf8-cli + + - name: Provision redis + env: + CF_USERNAME: ${{ secrets.CF_USERNAME }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD }} + CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }} + CF_SPACE: dev + CF_ORG: ${{ secrets.CF_ORG }} + run: | + cf api $CF_API_ENDPOINT + cf auth + cf target -o $CF_ORG -s $CF_SPACE + cf create-service redis micro-6.x dluhc-core-review-${{ github.event.pull_request.number }}-redis --wait + + deploy: + name: Deploy review app + runs-on: ubuntu-latest + environment: staging + needs: [postgres, redis] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Install Cloud Foundry CLI + run: | + wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - + echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list + sudo apt-get update + sudo apt-get install cf8-cli + + - name: Setup review app without starting + env: + CF_USERNAME: ${{ secrets.CF_USERNAME }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD }} + CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }} + CF_SPACE: dev + CF_ORG: ${{ secrets.CF_ORG }} + APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }} + run: | + cf api $CF_API_ENDPOINT + cf auth + cf target -o $CF_ORG -s $CF_SPACE + cf push $APP_NAME \ + --manifest ./config/cloud_foundry/review_manifest.yml \ + --no-start + + - name: Set environment variables + env: + APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }} + API_USER: ${{ secrets.API_USER }} + API_KEY: ${{ secrets.API_KEY }} + GOVUK_NOTIFY_API_KEY: ${{ secrets.GOVUK_NOTIFY_API_KEY }} + RAILS_MASTER_KEY: ${{ secrets.RAILS_MASTER_KEY }} + IMPORT_PAAS_INSTANCE: ${{ secrets.IMPORT_PAAS_INSTANCE }} + EXPORT_PAAS_INSTANCE: ${{ secrets.EXPORT_PAAS_INSTANCE }} + S3_CONFIG: ${{ secrets.S3_CONFIG }} + CSV_DOWNLOAD_PAAS_INSTANCE: ${{ secrets.CSV_DOWNLOAD_PAAS_INSTANCE }} + SENTRY_DSN: ${{ secrets.SENTRY_DSN }} + run: | + cf set-env $APP_NAME API_USER $API_USER + cf set-env $APP_NAME API_KEY $API_KEY + cf set-env $APP_NAME GOVUK_NOTIFY_API_KEY $GOVUK_NOTIFY_API_KEY + cf set-env $APP_NAME RAILS_MASTER_KEY $RAILS_MASTER_KEY + cf set-env $APP_NAME IMPORT_PAAS_INSTANCE $IMPORT_PAAS_INSTANCE + cf set-env $APP_NAME EXPORT_PAAS_INSTANCE $EXPORT_PAAS_INSTANCE + cf set-env $APP_NAME S3_CONFIG $S3_CONFIG + cf set-env $APP_NAME CSV_DOWNLOAD_PAAS_INSTANCE $CSV_DOWNLOAD_PAAS_INSTANCE + cf set-env $APP_NAME SENTRY_DSN $SENTRY_DSN + + - name: Bind postgres service + env: + APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }} + SERVICE_NAME: dluhc-core-review-${{ github.event.pull_request.number }}-postgres + run: | + cf bind-service $APP_NAME $SERVICE_NAME + + - name: Bind redis service + env: + APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }} + SERVICE_NAME: dluhc-core-review-${{ github.event.pull_request.number }}-redis + run: | + cf bind-service $APP_NAME $SERVICE_NAME + + - name: Bind logit drain service + env: + APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }} + SERVICE_NAME: logit-ssl-drain + run: | + cf bind-service $APP_NAME $SERVICE_NAME + + - name: Start review app + env: + APP_NAME: dluhc-core-review-${{ github.event.pull_request.number }} + run: | + cf restage $APP_NAME --strategy rolling + + - name: Comment on PR with URL + uses: unsplash/comment-on-pr@v1.3.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + msg: "Created review app at https://dluhc-core-review-${{ github.event.pull_request.number }}.london.cloudapps.digital" + check_for_duplicate_msg: true + duplicate_msg_pattern: Created review app at* diff --git a/.github/workflows/review_teardown_pipeline.yml b/.github/workflows/review_teardown_pipeline.yml new file mode 100644 index 000000000..37bdb9aa4 --- /dev/null +++ b/.github/workflows/review_teardown_pipeline.yml @@ -0,0 +1,92 @@ +name: Review app teardown pipeline + +on: + pull_request: + types: + - closed + workflow_dispatch: + +defaults: + run: + shell: bash + +jobs: + app: + name: Teardown app + runs-on: ubuntu-latest + environment: staging + + steps: + - name: Install Cloud Foundry CLI + run: | + wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - + echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list + sudo apt-get update + sudo apt-get install cf8-cli + + - name: Teardown app + env: + CF_USERNAME: ${{ secrets.CF_USERNAME }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD }} + CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }} + CF_SPACE: dev + CF_ORG: ${{ secrets.CF_ORG }} + run: | + cf api $CF_API_ENDPOINT + cf auth + cf target -o $CF_ORG -s $CF_SPACE + cf delete dluhc-core-review-${{ github.event.pull_request.number }} -f -r + + postgres: + name: Teardown postgres + runs-on: ubuntu-latest + environment: staging + needs: [app] + + steps: + - name: Install Cloud Foundry CLI + run: | + wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - + echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list + sudo apt-get update + sudo apt-get install cf8-cli + + - name: Teardown postgres + env: + CF_USERNAME: ${{ secrets.CF_USERNAME }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD }} + CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }} + CF_SPACE: dev + CF_ORG: ${{ secrets.CF_ORG }} + run: | + cf api $CF_API_ENDPOINT + cf auth + cf target -o $CF_ORG -s $CF_SPACE + cf delete-service dluhc-core-review-${{ github.event.pull_request.number }}-postgres --wait -f + + redis: + name: Teardown redis + runs-on: ubuntu-latest + environment: staging + needs: [app] + + steps: + - name: Install Cloud Foundry CLI + run: | + wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add - + echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list + sudo apt-get update + sudo apt-get install cf8-cli + + - name: Teardown redis + env: + CF_USERNAME: ${{ secrets.CF_USERNAME }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD }} + CF_API_ENDPOINT: ${{ secrets.CF_API_ENDPOINT }} + CF_SPACE: dev + CF_ORG: ${{ secrets.CF_ORG }} + run: | + cf api $CF_API_ENDPOINT + cf auth + cf target -o $CF_ORG -s $CF_SPACE + cf delete-service dluhc-core-review-${{ github.event.pull_request.number }}-redis --wait -f diff --git a/.rubocop.yml b/.rubocop.yml index 059bc2a84..87422dd67 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -27,3 +27,4 @@ Rails/UnknownEnv: - staging - development - test + - review diff --git a/app/models/user.rb b/app/models/user.rb index 40a430489..6126e7004 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -125,6 +125,7 @@ class User < ApplicationRecord def need_two_factor_authentication?(_request) return false if Rails.env.development? + return false if Rails.env.review? support? end diff --git a/config/cloud_foundry/review_manifest.yml b/config/cloud_foundry/review_manifest.yml new file mode 100644 index 000000000..f715306b0 --- /dev/null +++ b/config/cloud_foundry/review_manifest.yml @@ -0,0 +1,21 @@ +--- +defaults: &defaults + buildpacks: + - https://github.com/cloudfoundry/ruby-buildpack.git#v1.8.59 + processes: + - type: web + command: bundle exec rake cf:on_first_instance db:migrate db:seed && bin/rails server + instances: 1 + memory: 1G + - type: worker + command: bundle exec sidekiq + health-check-type: process + instances: 1 + health-check-type: http + health-check-http-endpoint: /health + +applications: + - name: dluhc-core-review + <<: *defaults + env: + RAILS_ENV: review diff --git a/config/database.yml b/config/database.yml index c09e3600a..b5563e716 100644 --- a/config/database.yml +++ b/config/database.yml @@ -31,6 +31,9 @@ development: database: <%= ENV['DB_DATABASE'] || 'data_collector_development' %> host: <%= ENV['DB_HOST'] || 'localhost' %> +review: + <<: *default + staging: <<: *default diff --git a/config/environments/review_app.rb b/config/environments/review_app.rb new file mode 100644 index 000000000..27b4a5c50 --- /dev/null +++ b/config/environments/review_app.rb @@ -0,0 +1,130 @@ +require "active_support/core_ext/integer/time" + +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # Code is not reloaded between requests. + config.cache_classes = true + + # Eager load code on boot. This eager loads most of Rails and + # your application in memory, allowing both threaded web servers + # and those relying on copy on write to perform better. + # Rake tasks automatically ignore this option for performance. + config.eager_load = true + + # Full error reports are disabled and caching is turned on. + config.consider_all_requests_local = false + config.action_controller.perform_caching = true + + # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"] + # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). + # config.require_master_key = true + + # Disable serving static files from the `/public` folder by default since + # Apache or NGINX already handles this. + config.public_file_server.enabled = ENV["RAILS_SERVE_STATIC_FILES"].present? + + # Enable serving of images, stylesheets, and JavaScripts from an asset server. + # config.asset_host = 'http://assets.example.com' + + # Specifies the header that your server uses for sending files. + # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache + # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + + # Store uploaded files on the local file system (see config/storage.yml for options). + config.active_storage.service = :local + + # Mount Action Cable outside main process or domain. + # config.action_cable.mount_path = nil + # config.action_cable.url = 'wss://example.com/cable' + # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] + + # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. + # config.force_ssl = true + + # Include generic and useful information about system operation, but avoid logging too much + # information to avoid inadvertent exposure of personally identifiable information (PII). + config.log_level = :info + + # Prepend all log lines with the following tags. + config.log_tags = [:request_id] + + # Use a different cache store in production. + # config.cache_store = :mem_cache_store + + # Use a real queuing backend for Active Job (and separate queues per environment). + # config.active_job.queue_adapter = :resque + # config.active_job.queue_name_prefix = "data_collector_production" + + config.action_mailer.perform_caching = false + + config.action_mailer.default_url_options = { host: ENV["APP_HOST"] } + config.action_mailer.delivery_method = :smtp + config.action_mailer.smtp_settings = { + address: "smtp.gmail.com", + port: 465, + domain: "gmail.com", + user_name: ENV["CORE_EMAIL_USERNAME"], + password: ENV["CORE_EMAIL_PASSWORD"], + authentication: "plain", + enable_starttls_auto: true, + ssl: true, + } + + # Ignore bad email addresses and do not raise email delivery errors. + # Set this to true and configure the email server for immediate delivery to raise delivery errors. + # config.action_mailer.raise_delivery_errors = false + + # Enable locale fallbacks for I18n (makes lookups for any locale fall back to + # the I18n.default_locale when a translation cannot be found). + config.i18n.fallbacks = true + + # Send deprecation notices to registered listeners. + config.active_support.deprecation = :notify + + # Log disallowed deprecations. + config.active_support.disallowed_deprecation = :log + + # Tell Active Support which deprecation messages to disallow. + config.active_support.disallowed_deprecation_warnings = [] + + # Use default logging formatter so that PID and timestamp are not suppressed. + config.log_formatter = ::Logger::Formatter.new + + # Use a different logger for distributed setups. + # require "syslog/logger" + # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') + + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new($stdout) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + end + + # Do not dump schema after migrations. + config.active_record.dump_schema_after_migration = false + + # Inserts middleware to perform automatic connection switching. + # The `database_selector` hash is used to pass options to the DatabaseSelector + # middleware. The `delay` is used to determine how long to wait after a write + # to send a subsequent read to the primary. + # + # The `database_resolver` class is used by the middleware to determine which + # database is appropriate to use based on the time delay. + # + # The `database_resolver_context` class is used by the middleware to set + # timestamps for the last write to the primary. The resolver uses the context + # class timestamps to determine how long to wait before reading from the + # replica. + # + # By default Rails will store a last write timestamp in the session. The + # DatabaseSelector middleware is designed as such you can define your own + # strategy for connection switching and pass that into the middleware through + # these configuration options. + # config.active_record.database_selector = { delay: 2.seconds } + # config.active_record.database_resolver = ActiveRecord::Middleware::DatabaseSelector::Resolver + # config.active_record.database_resolver_context = ActiveRecord::Middleware::DatabaseSelector::Resolver::Session + + # see https://discuss.rubyonrails.org/t/cve-2022-32224-possible-rce-escalation-bug-with-serialized-columns-in-active-record/81017 + config.active_record.yaml_column_permitted_classes = [Time] +end diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index bc484e77a..febd58a50 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -4,6 +4,9 @@ require "configuration/paas_configuration_service" if Rails.env.development? || Rails.env.test? Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new Rack::Attack.enabled = false +elsif Rails.env.review? + redis_url = Configuration::PaasConfigurationService.new.redis_uris.to_a[0][1] + Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(url: redis_url) else redis_url = Configuration::PaasConfigurationService.new.redis_uris[:"dluhc-core-#{Rails.env}-redis"] Rack::Attack.cache.store = ActiveSupport::Cache::RedisCacheStore.new(url: redis_url) diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb index 361739cbc..3b7262837 100644 --- a/config/initializers/sentry.rb +++ b/config/initializers/sentry.rb @@ -1,5 +1,5 @@ Sentry.init do |config| config.breadcrumbs_logger = %i[active_support_logger http_logger] - config.enabled_environments = %w[production staging] + config.enabled_environments = %w[production staging review] config.traces_sample_rate = 0.2 end diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index fa377de99..59e0064d8 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -11,3 +11,15 @@ if Rails.env.staging? || Rails.env.production? config.redis = { url: redis_url } end end + +if Rails.env.review? + redis_url = Configuration::PaasConfigurationService.new.redis_uris.to_a[0][1] + + Sidekiq.configure_server do |config| + config.redis = { url: redis_url } + end + + Sidekiq.configure_client do |config| + config.redis = { url: redis_url } + end +end diff --git a/db/seeds.rb b/db/seeds.rb index 9d78fd8cb..7bcc90abc 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -88,7 +88,7 @@ unless Rails.env.test? relationship_type: OrganisationRelationship::MANAGING, ) - if Rails.env.development? && User.count.zero? + if (Rails.env.development? || Rails.env.review?) && User.count.zero? User.create!( name: "Provider", email: "provider@example.com", @@ -119,7 +119,7 @@ unless Rails.env.test? pp "Seeded 3 dummy users" end - if Rails.env.development? + if Rails.env.development? || Rails.env.review? dummy_org = Organisation.find_or_create_by!( name: "FooBar LTD", address_line1: "Higher Kingston", @@ -134,7 +134,7 @@ unless Rails.env.test? pp "Seeded dummy FooBar LTD organisation" end - if Rails.env.development? && Scheme.count.zero? + if (Rails.env.development? || Rails.env.review?) && Scheme.count.zero? scheme1 = Scheme.create!( service_name: "Beulahside Care", sensitive: 0, diff --git a/docs/infrastructure.md b/docs/infrastructure.md index ca1324231..63bbf2e42 100644 --- a/docs/infrastructure.md +++ b/docs/infrastructure.md @@ -108,6 +108,20 @@ When a commit is made to `main` the following GitHub action jobs are triggered: When a pull request is opened to `main` only the Test stage runs. +## Review apps + +When a pull request is opened a review app will be spun up. The reviews apps connect to their own PostgreSQL and Redis instances with its own worker. + +The review app github pipeline is independent of any test pipeline and therefore it will attempt to deploy regardless of the state the code is in. + +The usual seeding process takes place when the review app boots so there will be some minimal data that can be used to login with. 2FA has been disabled in the review apps for easier access. + +The app boots in a new environment called `review`. As such this is the environment you should filter by for sentry errors or to change any config. + +After a sucessful deployment a comment will be added to the pull request with the URL to the review app for your convenience. When a pull request is updated e.g. more code is added it will re-deploy the new code. + +Once a pull request has been closed the review app infrastructure will be tore down to save on any costs. Should you wish to re-open a closed pull request the review app will be spun up again. + ## Setting up Infrastructure for a new environment ### Staging diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 450e334a7..bd6555348 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -159,6 +159,19 @@ RSpec.describe User, type: :model do expect(user.need_two_factor_authentication?(nil)).to be false end end + + context "when the user is in review environment" do + let(:user) { FactoryBot.create(:user, :support) } + + before do + allow(Rails.env).to receive(:development?).and_return(false) + allow(Rails.env).to receive(:review?).and_return(true) + end + + it "does not require 2FA" do + expect(user.need_two_factor_authentication?(nil)).to be false + end + end end describe "paper trail" do