From 28967f2fe4963328289903cdd20268298642bbec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Meny?= Date: Sat, 17 Feb 2024 16:10:39 +0000 Subject: [PATCH 1/5] Attempt at fixing development docker compose stack --- .env.development | 21 +++ .env.test | 13 ++ .github/workflows/deploy-aws-production.yml | 145 +------------------- .github/workflows/deploy-aws-staging.yml | 14 +- .gitignore | 1 - Dockerfile-prod => Dockerfile | 0 Dockerfile-dev | 28 ---- Makefile | 35 ++--- app/services/paas_configuration_service.rb | 9 +- app/services/storage_service.rb | 61 ++++---- bin/init-s3-localstack.py | 10 ++ config/environments/production.rb | 2 +- config/environments/staging.rb | 2 +- config/environments/test.rb | 6 + config/storage.yml | 23 ---- docker-compose.yml | 94 ++++--------- http-mock-server/Dockerfile | 2 +- web-manifest.yml | 41 ------ worker-manifest.yml | 27 ---- 19 files changed, 132 insertions(+), 402 deletions(-) create mode 100644 .env.development create mode 100644 .env.test rename Dockerfile-prod => Dockerfile (100%) delete mode 100644 Dockerfile-dev create mode 100644 bin/init-s3-localstack.py delete mode 100644 web-manifest.yml delete mode 100644 worker-manifest.yml diff --git a/.env.development b/.env.development new file mode 100644 index 00000000..c9ef7618 --- /dev/null +++ b/.env.development @@ -0,0 +1,21 @@ +RAILS_ENV=development +INSTANCE_NAME=ukraine-sponsor-resettlement-development +DB_HOST=pg +DB_USERNAME=ukraine +DB_DATABASE=ukraine_test +DB_PASSWORD=password +REDIS_URL=redis://redis +AWS_ACCESS_KEY_ID=test +AWS_SECRET_ACCESS_KEY=test +AWS_BUCKET_NAME=test-bucket +AWS_REGION=eu-west-2 +REMOTE_API_URL=http://mock-api:8081/data +BASIC_AUTH_PASS=password +UAM_GA4_TRACKING_CODE=G-ZT5Q8BV3ZK +EOI_GA4_TRACKING_CODE=G-W9KQ85GQM4 +INDIVIDUAL_CONFIRMATION_TEMPLATE_ID=6c80930d-e25a-4dc6-8383-bb83a2c18d19 +ORGANISATION_CONFIRMATION_TEMPLATE_ID=3ae2501f-e2be-4ad6-886b-fcf5aa71d448 +ADDITIONAL_INFO_CONFIRMATION_TEMPLATE_ID=5b79f75e-fff3-4585-a438-d59966b86dd9 +SPONSOR_CONFIRMATION_TEMPLATE_ID=f2a29524-be50-4fc5-9088-35daf4c66c43 +SAVE_AND_RETURN_TEMPLATE_ID=856b0664-1f96-463c-8acd-c4ef782cdcad +GOVUK_NOTIFY_API_KEY=test diff --git a/.env.test b/.env.test new file mode 100644 index 00000000..dbd3cc7c --- /dev/null +++ b/.env.test @@ -0,0 +1,13 @@ +RAILS_ENV=test +INSTANCE_NAME=ukraine-sponsor-resettlement-test +DB_HOST=localhost +DB_USERNAME=ukraine +DB_DATABASE=ukraine_test +DB_PASSWORD=password +REDIS_URL=redis://redis +AWS_ACCESS_KEY_ID=test +AWS_SECRET_ACCESS_KEY=test +AWS_BUCKET_NAME=test-bucket +AWS_REGION=eu-west-2 +REMOTE_API_URL=http://mock-api:8081/data +GOVUK_NOTIFY_API_KEY=test diff --git a/.github/workflows/deploy-aws-production.yml b/.github/workflows/deploy-aws-production.yml index b244cd46..7b8caedd 100644 --- a/.github/workflows/deploy-aws-production.yml +++ b/.github/workflows/deploy-aws-production.yml @@ -10,10 +10,6 @@ name: Deploy to Amazon ECS PRODUCTION on: - # pull_request: - # push: - # branches: - # - master workflow_dispatch: inputs: refToDeploy: @@ -37,149 +33,10 @@ env: # containerDefinitions section of your task definition jobs: - # test: - # name: Test - # runs-on: ubuntu-latest - - # services: - # postgres: - # image: postgres:13.5 - # env: - # POSTGRES_PASSWORD: password - # POSTGRES_USER: ukraine - # POSTGRES_DB: ukraine - # ports: - # - 5432:5432 - # # needed because the postgres container does not provide a healthcheck - # # tmpfs makes DB faster by using RAM - # options: >- - # --mount type=tmpfs,destination=/var/lib/postgresql/data - # --health-cmd pg_isready - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - # redis: - # # Docker Hub image - # image: redis - # # Set health checks to wait until redis has started - # options: >- - # --health-cmd "redis-cli ping" - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - - # env: - # RAILS_ENV: test - # INSTANCE_NAME: ukraine-sponsor-resettlement-test - # VCAP_SERVICES: '{"redis":[{"instance_name":"ukraine-sponsor-resettlement-test-redis","credentials":{"uri":"redis://redis"}}],"aws-s3-bucket":[{"instance_name":"ukraine-sponsor-resettlement-test-s3","credentials":{"aws_access_key_id":"access-key-id","aws_secret_access_key":"secret-access-key","aws_region":"eu-west-2","bucket_name":"test-bucket"}}]}' - # GEMFILE_RUBY_VERSION: 3.1.2 - # DB_HOST: localhost - # DB_DATABASE: ukraine - # DB_USERNAME: ukraine - # DB_PASSWORD: password - # GOVUK_NOTIFY_API_KEY: ${{ secrets.GOVUK_NOTIFY_API_KEY }} - - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # ref: ${{ inputs.refToDeploy }} - - # - name: Set up Ruby - # uses: ruby/setup-ruby@v1 - # with: - # # runs 'bundle install' and caches installed gems automatically - # bundler-cache: true - # - name: Set up node - # uses: actions/setup-node@v4 - # with: - # node-version: "20" - - # - name: Create DB - # run: | - # bundle exec rake db:prepare - # - name: Migrate DB - # run: | - # bundle exec rake db:migrate - # - name: Compile Assets - # run: | - # bundle exec rake assets:precompile - # - name: Run tests - # run: | - # bundle exec rake - # #- name: Test coverage - # # uses: joshmfrankel/simplecov-check-action@main - # # with: - # # github_token: ${{ secrets.GITHUB_TOKEN }} - # # minimum_suite_coverage: 70 - # # minimum_file_coverage: 0 - - # lint: - # name: Lint - # runs-on: ubuntu-latest - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # ref: ${{ inputs.refToDeploy }} - - # - name: Set up Ruby - # uses: ruby/setup-ruby@v1 - # with: - # # runs 'bundle install' and caches installed gems automatically - # bundler-cache: true - - # - name: Rubocop - # run: | - # bundle exec rubocop - - # audit: - # name: Audit dependencies - # runs-on: ubuntu-latest - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # ref: ${{ inputs.refToDeploy }} - - # - name: Set up Ruby - # uses: ruby/setup-ruby@v1 - # with: - # # runs 'bundle install' and caches installed gems automatically - # bundler-cache: true - - # - name: Audit - # run: | - # bundle exec bundler-audit - - # brakeman-scan: - # name: Brakeman Scan - # runs-on: ubuntu-latest - # steps: - # # Checkout the repository to the GitHub Actions runner - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # ref: ${{ inputs.refToDeploy }} - - # # Customize the ruby version depending on your needs - # - name: Setup Ruby - # uses: ruby/setup-ruby@v1 - - # - name: Setup Brakeman - # run: | - # gem install brakeman - - # # Execute Brakeman CLI and generate a SARIF output with the security issues identified during the analysis - # - name: Scan - # run: | - # brakeman --color -o /dev/stdout - deploy: name: Deploy runs-on: ubuntu-latest environment: aws-production - # needs: [lint, test, audit, brakeman-scan] steps: - name: Checkout @@ -206,7 +63,7 @@ jobs: # Build a docker container and # push it to ECR so that it can # be deployed to ECS. - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile-prod . + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile . docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT diff --git a/.github/workflows/deploy-aws-staging.yml b/.github/workflows/deploy-aws-staging.yml index 8b5522b0..0c5da821 100644 --- a/.github/workflows/deploy-aws-staging.yml +++ b/.github/workflows/deploy-aws-staging.yml @@ -40,9 +40,9 @@ jobs: postgres: image: postgres:13.5 env: - POSTGRES_PASSWORD: password + POSTGRES_DB: ukraine_test POSTGRES_USER: ukraine - POSTGRES_DB: ukraine + POSTGRES_PASSWORD: password ports: - 5432:5432 # needed because the postgres container does not provide a healthcheck @@ -69,7 +69,7 @@ jobs: VCAP_SERVICES: '{"redis":[{"instance_name":"ukraine-sponsor-resettlement-test-redis","credentials":{"uri":"redis://redis"}}],"aws-s3-bucket":[{"instance_name":"ukraine-sponsor-resettlement-test-s3","credentials":{"aws_access_key_id":"access-key-id","aws_secret_access_key":"secret-access-key","aws_region":"eu-west-2","bucket_name":"test-bucket"}}]}' GEMFILE_RUBY_VERSION: 3.1.4 DB_HOST: localhost - DB_DATABASE: ukraine + DB_DATABASE: ukraine_test DB_USERNAME: ukraine DB_PASSWORD: password GOVUK_NOTIFY_API_KEY: ${{ secrets.GOVUK_NOTIFY_API_KEY }} @@ -102,12 +102,6 @@ jobs: - name: Run tests run: | bundle exec rake - #- name: Test coverage - # uses: joshmfrankel/simplecov-check-action@main - # with: - # github_token: ${{ secrets.GITHUB_TOKEN }} - # minimum_suite_coverage: 70 - # minimum_file_coverage: 0 lint: name: Lint @@ -201,7 +195,7 @@ jobs: # Build a docker container and # push it to ECR so that it can # be deployed to ECS. - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile-prod . + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile . docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT diff --git a/.gitignore b/.gitignore index 93cb47d8..204fd3ee 100644 --- a/.gitignore +++ b/.gitignore @@ -34,7 +34,6 @@ config/master.key # dotenv, dotenv-rails # TODO Comment out these rules if environment variables can be committed .env -.env*.local ## Environment normalization: /.bundle diff --git a/Dockerfile-prod b/Dockerfile similarity index 100% rename from Dockerfile-prod rename to Dockerfile diff --git a/Dockerfile-dev b/Dockerfile-dev deleted file mode 100644 index 446e35a6..00000000 --- a/Dockerfile-dev +++ /dev/null @@ -1,28 +0,0 @@ -# Build compilation image -FROM ruby:3.1.4-alpine3.18 AS devbuild - -# The application runs from /app -WORKDIR /app - -# Add the timezone as it's not configured by default in Alpine -RUN apk add --update --no-cache tzdata && cp /usr/share/zoneinfo/Europe/London /etc/localtime && echo "Europe/London" > /etc/timezone - -RUN apk add --no-cache build-base yarn postgresql-dev git bash gcompat - -ADD Gemfile /app/Gemfile -ADD Gemfile.lock /app/Gemfile.lock - -RUN gem install bundler:2.3.22 --no-document && \ - bundle install && \ - yarn install - -ENV PORT=8080 - -CMD RAILS_ENV=${RAILS_ENV} rm -f /app/tmp/pids/server.pid && bundle exec rake db:migrate && bundle exec rails s -e ${RAILS_ENV} -p ${PORT} --binding=0.0.0.0 - -EXPOSE ${PORT} - -# For running the tests, copy the app into the container because they run too slowly from a bind mount -FROM devbuild AS testingbuild -COPY . /app - diff --git a/Makefile b/Makefile index a73f8343..c6f4edc6 100644 --- a/Makefile +++ b/Makefile @@ -1,33 +1,14 @@ .PHONY: run run: - docker-compose up -d - -.PHONY: run-fg -run-fg: - docker-compose up -d pg - docker-compose up -d redis - docker-compose up --build app - -.PHONY: stop -stop: - docker-compose down + # Start all dependencies using development configuration + docker-compose --env-file .env.development up --detach .PHONY: test test: - docker-compose up --build test + # Start a test container running against existing dependencies + docker-compose --profile test up --build --force-recreate test -.PHONY: cf -cf: - docker run -it --mount src=`pwd`,target=/home/piper/app,type=bind ppiper/cf-cli:latest /bin/bash - -.PHONY: test-local -test-local: - RAILS_ENV=test \ - INSTANCE_NAME=ukraine-sponsor-resettlement-test \ - DB_HOST=localhost \ - DB_USERNAME=ukraine \ - DB_DATABASE=ukraine_test \ - DB_PASSWORD=password \ - VCAP_SERVICES='{"redis":[{"instance_name":"ukraine-sponsor-resettlement-test-redis","credentials":{"uri":"redis://redis"}}],"aws-s3-bucket":[{"instance_name":"ukraine-sponsor-resettlement-test-s3","access_key_id":"access-key-id","secret_access_key":"secret-access-key","region":"eu-west-2"}]}' \ - REMOTE_API_URL=http://mock-api:8081/data \ - bundle exec rspec +.PHONY: stop +stop: + # Stop all containers belonging to the project stack + docker-compose --env-file .env.development down diff --git a/app/services/paas_configuration_service.rb b/app/services/paas_configuration_service.rb index 5c291947..d97b5514 100644 --- a/app/services/paas_configuration_service.rb +++ b/app/services/paas_configuration_service.rb @@ -31,21 +31,22 @@ def read_paas_config end begin + @logger.warn("The use of VCAP_SERVICES is deprecated post GOV.UK PaaS migration") JSON.parse(ENV["VCAP_SERVICES"], { symbolize_names: true }) rescue StandardError @logger.warn("Could not parse VCAP_SERVICES") {} end else - c = {} - c[:redis] = [{ + configuration = {} + configuration[:redis] = [{ instance_name: "#{ENV['INSTANCE_NAME']}-redis", credentials: { uri: ENV["REDIS_URL"], }, }] - c[:'aws-s3-bucket'] = [{ + configuration[:'aws-s3-bucket'] = [{ instance_name: "#{ENV['INSTANCE_NAME']}-s3", credentials: { aws_access_key_id: ENV["AWS_ACCESS_KEY_ID"], @@ -54,7 +55,7 @@ def read_paas_config aws_region: ENV["AWS_REGION"], }, }] - c + configuration end end diff --git a/app/services/storage_service.rb b/app/services/storage_service.rb index 0f232868..99296bb1 100644 --- a/app/services/storage_service.rb +++ b/app/services/storage_service.rb @@ -1,29 +1,21 @@ class StorageService attr_reader :configuration - def initialize(paas_config_service, paas_instance_name) - @paas_config_service = paas_config_service - @paas_instance_name = ("#{paas_instance_name}-s3" || "").to_sym + def initialize(config_service, instance_name) + @config_service = config_service + @instance_name = "#{instance_name}-s3".to_sym @configuration = create_configuration @client = create_client end def write_file(file_name, data) - # rubocop:disable Style/RedundantBegin - # rubocop:disable Style/RescueStandardError - begin - @client.put_object( - body: data, - bucket: @configuration.bucket_name, - key: file_name, - ) - rescue - # Do nothing for now! - # TODO remove try...catch - Rails.logger.debug "Could NOT upload file!" - end - # rubocop:enable Style/RedundantBegin - # rubocop:enable Style/RescueStandardError + @client.put_object( + body: data, + bucket: @configuration.bucket_name, + key: file_name, + ) + rescue StandardError => e + Rails.logger.error "Could not upload file #{file_name}, #{e.message}" end def download(object_key) @@ -45,25 +37,36 @@ def download(object_key) private def create_configuration - unless @paas_config_service.config_present? + unless @config_service.config_present? raise "No PaaS configuration present" end - unless @paas_config_service.s3_buckets.key?(@paas_instance_name) - raise "#{@paas_instance_name} instance name could not be found" + unless @config_service.s3_buckets.key?(@instance_name) + raise "#{@instance_name} instance name could not be found" end - bucket_config = @paas_config_service.s3_buckets[@paas_instance_name] + bucket_config = @config_service.s3_buckets[@instance_name] StorageConfiguration.new(bucket_config[:credentials]) end def create_client - Aws::S3::Client.new( - region: @configuration.region, - credentials: Aws::Credentials.new( - @configuration.access_key_id, - @configuration.secret_access_key, - ), - ) + if Rails.env.test? + Aws::S3::Client.new( + endpoint: "http://localhost:4566", + credentials: Aws::Credentials.new( + @configuration.access_key_id, + @configuration.secret_access_key, + ), + force_path_style: true, + ) + else + Aws::S3::Client.new( + region: @configuration.region, + credentials: Aws::Credentials.new( + @configuration.access_key_id, + @configuration.secret_access_key, + ), + ) + end end end diff --git a/bin/init-s3-localstack.py b/bin/init-s3-localstack.py new file mode 100644 index 00000000..5720d2a9 --- /dev/null +++ b/bin/init-s3-localstack.py @@ -0,0 +1,10 @@ +import boto3 + +s3_client = boto3.client( + "s3", + endpoint_url=f"http://localhost:4566", + aws_access_key_id="test", + aws_secret_access_key="test" +) + +s3_client.create_bucket(Bucket="test-bucket") diff --git a/config/environments/production.rb b/config/environments/production.rb index feff5c9c..7b77e7a5 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -47,7 +47,7 @@ # Use the lowest log level to ensure availability of diagnostic information # when problems arise. - config.log_level = :debug + config.log_level = :info # Prepend all log lines with the following tags. config.log_tags = [:request_id] diff --git a/config/environments/staging.rb b/config/environments/staging.rb index feff5c9c..7b77e7a5 100644 --- a/config/environments/staging.rb +++ b/config/environments/staging.rb @@ -47,7 +47,7 @@ # Use the lowest log level to ensure availability of diagnostic information # when problems arise. - config.log_level = :debug + config.log_level = :info # Prepend all log lines with the following tags. config.log_tags = [:request_id] diff --git a/config/environments/test.rb b/config/environments/test.rb index eeaa5b8c..a2f1c46c 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -47,4 +47,10 @@ # Raises error for missing translations # config.action_view.raise_on_missing_translations = true config.active_job.queue_adapter = :test + + # Logs all warnings and errors to the standard output in tests + logger = ActiveSupport::Logger.new($stdout) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + config.log_level = :warn end diff --git a/config/storage.yml b/config/storage.yml index 03fbc04b..f17cd672 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -1,8 +1,4 @@ test: - service: Disk - root: <%= Rails.root.join("tmp/storage") %> - -local: service: Disk root: <%= Rails.root.join("storage") %> @@ -13,22 +9,3 @@ amazon: secret_access_key: <%= ENV["AWS_SECRET_ACCESS_KEY"] %> region: eu-west-2 bucket: <%= ENV["AWS_BUCKET_NAME"] %> - -# Remember not to checkin your GCS keyfile to a repository -# google: -# service: GCS -# project: your_project -# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %> -# bucket: your_own_bucket - -# Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key) -# microsoft: -# service: AzureStorage -# storage_account_name: your_account_name -# storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %> -# container: your_container_name - -# mirror: -# service: Mirror -# primary: local -# mirrors: [ amazon, google, microsoft ] diff --git a/docker-compose.yml b/docker-compose.yml index 2d884020..e599efda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,111 +1,75 @@ -version: "3.3" +version: "3.7" services: pg: image: postgres:13.5-alpine restart: always environment: - POSTGRES_PASSWORD: password - POSTGRES_DB: ukraine - POSTGRES_USER: ukraine + - POSTGRES_DB=${DB_DATABASE} + - POSTGRES_USER=${DB_USERNAME} + - POSTGRES_PASSWORD=${DB_PASSWORD} ports: - - 5432:5432 + - "127.0.0.1:5432:5432" app: - build: - dockerfile: Dockerfile-dev - context: . - target: devbuild + image: ukraine-sponsor-resettlement + pull_policy: never depends_on: - pg - redis - sidekiq - s3 - volumes: - - .:/app:cached # For development, the app dir is a bind mount to the dir on the host environment: + # Rails will use existing .env.development file RAILS_ENV: development - INSTANCE_NAME: ukraine-sponsor-resettlement-development - DB_HOST: pg - DB_DATABASE: ukraine - DB_USERNAME: ukraine - DB_PASSWORD: password - VCAP_SERVICES: '{"redis":[{"instance_name":"ukraine-sponsor-resettlement-development-redis","credentials":{"uri":"redis://redis"}}],"aws-s3-bucket":[{"instance_name":"ukraine-sponsor-resettlement-development-s3","credentials":{"aws_access_key_id":"access-key-id","aws_secret_access_key":"secret-access-key","aws_region":"eu-west-2","bucket_name":"upload-bucket"}}]}' - BASIC_AUTH_PASS: password - UAM_GA4_TRACKING_CODE: "G-ZT5Q8BV3ZK" - EOI_GA4_TRACKING_CODE: "G-W9KQ85GQM4" - ports: - - 8080:8080 + - "127.0.0.1:8080:8080" + command: > + sh -c "RAILS_ENV=${RAILS_ENV} rm -f /app/tmp/pids/server.pid && + bundle exec rake db:migrate && + bundle exec rails s -e ${RAILS_ENV} -p 8080 --binding=0.0.0.0" test: - build: - dockerfile: Dockerfile-dev - context: . - target: testingbuild # For running the tests the app files are copied into the container because running them from a bind mount is too slow + image: ukraine-sponsor-resettlement + pull_policy: never depends_on: - pg - redis + - sidekiq - s3 + profiles: ["test"] environment: + # Rails will use existing .env.test file RAILS_ENV: test - INSTANCE_NAME: ukraine-sponsor-resettlement-test - DB_HOST: pg - DB_USERNAME: ukraine - DB_DATABASE: ukraine_test - DB_PASSWORD: password - VCAP_SERVICES: '{"redis":[{"instance_name":"ukraine-sponsor-resettlement-test-redis","credentials":{"uri":"redis://redis"}}],"aws-s3-bucket":[{"instance_name":"ukraine-sponsor-resettlement-test-s3","credentials":{"aws_access_key_id":"access-key-id","aws_secret_access_key":"secret-access-key","aws_region":"eu-west-2","bucket_name":"test-bucket"}}]}' - REMOTE_API_URL: http://mock-api:8081/data - GOVUK_NOTIFY_API_KEY: ${GOVUK_NOTIFY_API_KEY} - command: bin/test.sh sidekiq: - build: - dockerfile: Dockerfile-dev - context: . - target: devbuild + build: . + image: ukraine-sponsor-resettlement + pull_policy: never depends_on: - pg - redis - mock-api - volumes: - - .:/app environment: - RAILS_ENV: test - DB_HOST: pg - DB_DATABASE: ukraine - DB_USERNAME: ukraine - DB_PASSWORD: password - VCAP_SERVICES: '{"redis":[{"instance_name":"ukraine-sponsor-resettlement-test-redis","credentials":{"uri":"redis://redis"}}],"aws-s3-bucket":[{"instance_name":"ukraine-sponsor-resettlement-test-s3","credentials":{"aws_access_key_id":"access-key-id","aws_secret_access_key":"secret-access-key","aws_region":"eu-west-2","bucket_name":"test-bucket"}}]}' - REMOTE_API_URL: http://mock-api:8081/data - INDIVIDUAL_CONFIRMATION_TEMPLATE_ID: "6c80930d-e25a-4dc6-8383-bb83a2c18d19" - ORGANISATION_CONFIRMATION_TEMPLATE_ID: "3ae2501f-e2be-4ad6-886b-fcf5aa71d448" - ADDITIONAL_INFO_CONFIRMATION_TEMPLATE_ID: "5b79f75e-fff3-4585-a438-d59966b86dd9" - SPONSOR_CONFIRMATION_TEMPLATE_ID: "f2a29524-be50-4fc5-9088-35daf4c66c43" - SAVE_AND_RETURN_TEMPLATE_ID: "856b0664-1f96-463c-8acd-c4ef782cdcad" - GOVUK_NOTIFY_API_KEY: ${GOVUK_NOTIFY_API_KEY} + # Rails will use existing .env.development file + RAILS_ENV: development command: bundle exec sidekiq redis: image: redis s3: - image: localstack/localstack:latest + image: localstack/localstack:s3-latest container_name: localstack-s3 environment: - - SERVICES=s3:5002 - - DEFAULT_REGION=eu-west-2 - - DATA_DIR=/tmp/localstack/data + - PERSISTENCE=1 ports: - - 5002:5002 - - 9999:8080 + - "127.0.0.1:4566:4566" volumes: - - .:/tmp/localstack + - "./bin/init-s3-localstack.py:/etc/localstack/init/ready.d/init-s3.py" mock-api: - build: - dockerfile: Dockerfile - context: http-mock-server + build: http-mock-server ports: - - 8081:8081 + - "127.0.0.1:8081:8081" diff --git a/http-mock-server/Dockerfile b/http-mock-server/Dockerfile index 20149308..0598230e 100644 --- a/http-mock-server/Dockerfile +++ b/http-mock-server/Dockerfile @@ -1,7 +1,7 @@ FROM ruby:3.1.4-alpine3.18 -RUN gem install sinatra webrick +RUN gem install sinatra webrick rackup COPY server.rb . CMD ruby server.rb -p 8081 -o 0.0.0.0 diff --git a/web-manifest.yml b/web-manifest.yml deleted file mode 100644 index bdfd4aba..00000000 --- a/web-manifest.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -defaults: &defaults - buildpacks: - - https://github.com/cloudfoundry/ruby-buildpack.git - command: bundle exec rake cf:on_first_instance db:migrate && bin/rails server - instances: 2 - memory: 512M - health-check-type: http - health-check-http-endpoint: /health - -applications: - - name: ukraine-sponsor-resettlement-test - <<: *defaults - env: - RAILS_ENV: test - INSTANCE_NAME: ukraine-sponsor-resettlement-test - services: - - ukraine-sponsor-resettlement-test-postgres - - ukraine-sponsor-resettlement-test-s3 - - - name: ukraine-sponsor-resettlement-staging - <<: *defaults - env: - RAILS_ENV: staging - INSTANCE_NAME: ukraine-sponsor-resettlement-staging - services: - - ukraine-sponsor-resettlement-staging-postgres - - ukraine-sponsor-resettlement-staging-redis - - ukraine-sponsor-resettlement-staging-s3 - - - name: ukraine-sponsor-resettlement-production - <<: *defaults - env: - RAILS_ENV: production - INSTANCE_NAME: ukraine-sponsor-resettlement-production - instances: 4 - memory: 512M - services: - - ukraine-sponsor-resettlement-production-postgres - - ukraine-sponsor-resettlement-production-redis - - ukraine-sponsor-resettlement-production-s3 diff --git a/worker-manifest.yml b/worker-manifest.yml deleted file mode 100644 index 5c3a20c6..00000000 --- a/worker-manifest.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -defaults: &defaults - buildpacks: - - https://github.com/cloudfoundry/ruby-buildpack.git - command: bundle exec sidekiq - instances: 2 - memory: 1G - health-check-type: process - no-route: true - -applications: - - name: ukraine-sponsor-resettlement-staging-sidekiq - <<: *defaults - env: - RAILS_ENV: staging - services: - - ukraine-sponsor-resettlement-staging-postgres - - ukraine-sponsor-resettlement-staging-redis - - - name: ukraine-sponsor-resettlement-production-sidekiq - <<: *defaults - env: - RAILS_ENV: production - instances: 2 - services: - - ukraine-sponsor-resettlement-production-postgres - - ukraine-sponsor-resettlement-production-redis From f0e677f91388e718976b6334d20536d64a3d59d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Meny?= Date: Sun, 18 Feb 2024 13:46:09 +0000 Subject: [PATCH 2/5] Runs local development server and spec from Makefile --- .env.development | 2 +- Makefile | 12 ++++++++++-- docker-compose.yml | 30 ++++++------------------------ 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/.env.development b/.env.development index c9ef7618..7fd8962c 100644 --- a/.env.development +++ b/.env.development @@ -1,6 +1,6 @@ RAILS_ENV=development INSTANCE_NAME=ukraine-sponsor-resettlement-development -DB_HOST=pg +DB_HOST=localhost DB_USERNAME=ukraine DB_DATABASE=ukraine_test DB_PASSWORD=password diff --git a/Makefile b/Makefile index c6f4edc6..3f5f8d21 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,22 @@ .PHONY: run run: # Start all dependencies using development configuration - docker-compose --env-file .env.development up --detach + docker-compose up --detach + RAILS_ENV=development bundle exec rake db:migrate && bundle exec rails s -p 8080 --binding=0.0.0.0 .PHONY: test test: # Start a test container running against existing dependencies + docker-compose up --detach + RAILS_ENV=test bundle exec rake db:migrate && bundle exec rake spec + +.PHONY: docker-test +docker-test: + # Start a test container running against existing dependencies + docker-compose up --detach docker-compose --profile test up --build --force-recreate test .PHONY: stop stop: # Stop all containers belonging to the project stack - docker-compose --env-file .env.development down + docker-compose down diff --git a/docker-compose.yml b/docker-compose.yml index e599efda..d66208c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,29 +5,11 @@ services: image: postgres:13.5-alpine restart: always environment: - - POSTGRES_DB=${DB_DATABASE} - - POSTGRES_USER=${DB_USERNAME} - - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=ukraine_test + - POSTGRES_USER=ukraine + - POSTGRES_PASSWORD=password ports: - - "127.0.0.1:5432:5432" - - app: - image: ukraine-sponsor-resettlement - pull_policy: never - depends_on: - - pg - - redis - - sidekiq - - s3 - environment: - # Rails will use existing .env.development file - RAILS_ENV: development - ports: - - "127.0.0.1:8080:8080" - command: > - sh -c "RAILS_ENV=${RAILS_ENV} rm -f /app/tmp/pids/server.pid && - bundle exec rake db:migrate && - bundle exec rails s -e ${RAILS_ENV} -p 8080 --binding=0.0.0.0" + - "5432:5432" test: image: ukraine-sponsor-resettlement @@ -65,11 +47,11 @@ services: environment: - PERSISTENCE=1 ports: - - "127.0.0.1:4566:4566" + - "4566:4566" volumes: - "./bin/init-s3-localstack.py:/etc/localstack/init/ready.d/init-s3.py" mock-api: build: http-mock-server ports: - - "127.0.0.1:8081:8081" + - "8081:8081" From f6a2025c14fe488df8a7a1f43e50789ea9936b23 Mon Sep 17 00:00:00 2001 From: baarkerlounger Date: Fri, 1 Mar 2024 13:08:56 +0000 Subject: [PATCH 3/5] Fix redis connection --- .env.development | 2 +- docker-compose.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.env.development b/.env.development index 7fd8962c..3d6359e5 100644 --- a/.env.development +++ b/.env.development @@ -4,7 +4,7 @@ DB_HOST=localhost DB_USERNAME=ukraine DB_DATABASE=ukraine_test DB_PASSWORD=password -REDIS_URL=redis://redis +REDIS_URL=redis://0.0.0.0:6379 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_BUCKET_NAME=test-bucket diff --git a/docker-compose.yml b/docker-compose.yml index d66208c8..b3fb0299 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,8 @@ services: redis: image: redis + ports: + - "6379:6379" s3: image: localstack/localstack:s3-latest From e5ffca9790c56a245aec860f962b3cd0fffeb7b7 Mon Sep 17 00:00:00 2001 From: mandanakhademi Date: Mon, 4 Mar 2024 15:06:50 +0000 Subject: [PATCH 4/5] Fix e2e tests --- automated_tests/cypress/e2e/page_elements/UAM/uam_elements.js | 1 + automated_tests/cypress/e2e/pages/UAM/uam_links.js | 2 +- automated_tests/cypress/e2e/pages/UAM/uam_your_details.js | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/automated_tests/cypress/e2e/page_elements/UAM/uam_elements.js b/automated_tests/cypress/e2e/page_elements/UAM/uam_elements.js index d1e56f52..26dee361 100644 --- a/automated_tests/cypress/e2e/page_elements/UAM/uam_elements.js +++ b/automated_tests/cypress/e2e/page_elements/UAM/uam_elements.js @@ -155,6 +155,7 @@ var page_elements = { step16_id_err_msg: "#unaccompanied-minor-identification-type-error", step16_idh_btn: "#unaccompanied-minor-identification-type-none-field", step16_pp_radio_btn: "#unaccompanied-minor-identification-type-passport-field", + step16_pp_err_radio_btn: "#unaccompanied-minor-identification-type-field-error", step16_ni_radio_btn: "#unaccompanied-minor-identification-type-national-identity-card-field", step16_biom_radio_btn: "#unaccompanied-minor-identification-type-biometric-residence-field", step16_pdl_radio_btn: "#unaccompanied-minor-identification-type-photo-driving-licence-field", diff --git a/automated_tests/cypress/e2e/pages/UAM/uam_links.js b/automated_tests/cypress/e2e/pages/UAM/uam_links.js index c7a41e5b..189ffd91 100644 --- a/automated_tests/cypress/e2e/pages/UAM/uam_links.js +++ b/automated_tests/cypress/e2e/pages/UAM/uam_links.js @@ -128,7 +128,7 @@ export const uam_cannot_uts_spon_alknow = () => { cy.visit('/sponsor-a-child/non-eligible') cannot_uts_heading() cy.xpath(elements.spon_alknow_link).invoke('removeAttr', 'target').click().wait(Cypress.env('waitTime')) - cy.get(elements.main_heading).contains('Apply for a visa under the Ukraine Sponsorship Scheme (Homes for Ukraine)').should('be.visible') + cy.get(elements.main_heading).contains('Apply for a visa under the Homes for Ukraine Sponsorship Scheme').should('be.visible') cy.url().should('include', link8).should('exist') cy.go('back') } diff --git a/automated_tests/cypress/e2e/pages/UAM/uam_your_details.js b/automated_tests/cypress/e2e/pages/UAM/uam_your_details.js index e408b13a..ba27d886 100644 --- a/automated_tests/cypress/e2e/pages/UAM/uam_your_details.js +++ b/automated_tests/cypress/e2e/pages/UAM/uam_your_details.js @@ -509,7 +509,7 @@ export const your_details_ad_details_id_step_16_v1 = () => { } //passport [min requirement 1 character] export const your_details_ad_details_id_step_16_v2 = () => { - cy.get(elements.step16_pp_radio_btn).click() + cy.get(elements.step16_pp_err_radio_btn).click() click_continue() pp_error() } From fe642f28622f58b58bb55f1e9709057f37609222 Mon Sep 17 00:00:00 2001 From: mandanakhademi Date: Mon, 4 Mar 2024 15:31:42 +0000 Subject: [PATCH 5/5] Update gem packages to fix issue with pipeline --- Gemfile.lock | 164 +++++++++++++++++++++++++-------------------------- 1 file changed, 80 insertions(+), 84 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 08985242..3be884f4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,35 +1,35 @@ GEM remote: https://rubygems.org/ specs: - actioncable (7.1.3) - actionpack (= 7.1.3) - activesupport (= 7.1.3) + actioncable (7.1.3.2) + actionpack (= 7.1.3.2) + activesupport (= 7.1.3.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.1.3) - actionpack (= 7.1.3) - activejob (= 7.1.3) - activerecord (= 7.1.3) - activestorage (= 7.1.3) - activesupport (= 7.1.3) + actionmailbox (7.1.3.2) + actionpack (= 7.1.3.2) + activejob (= 7.1.3.2) + activerecord (= 7.1.3.2) + activestorage (= 7.1.3.2) + activesupport (= 7.1.3.2) mail (>= 2.7.1) net-imap net-pop net-smtp - actionmailer (7.1.3) - actionpack (= 7.1.3) - actionview (= 7.1.3) - activejob (= 7.1.3) - activesupport (= 7.1.3) + actionmailer (7.1.3.2) + actionpack (= 7.1.3.2) + actionview (= 7.1.3.2) + activejob (= 7.1.3.2) + activesupport (= 7.1.3.2) mail (~> 2.5, >= 2.5.4) net-imap net-pop net-smtp rails-dom-testing (~> 2.2) - actionpack (7.1.3) - actionview (= 7.1.3) - activesupport (= 7.1.3) + actionpack (7.1.3.2) + actionview (= 7.1.3.2) + activesupport (= 7.1.3.2) nokogiri (>= 1.8.5) racc rack (>= 2.2.4) @@ -37,35 +37,35 @@ GEM rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actiontext (7.1.3) - actionpack (= 7.1.3) - activerecord (= 7.1.3) - activestorage (= 7.1.3) - activesupport (= 7.1.3) + actiontext (7.1.3.2) + actionpack (= 7.1.3.2) + activerecord (= 7.1.3.2) + activestorage (= 7.1.3.2) + activesupport (= 7.1.3.2) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.1.3) - activesupport (= 7.1.3) + actionview (7.1.3.2) + activesupport (= 7.1.3.2) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.3) - activesupport (= 7.1.3) + activejob (7.1.3.2) + activesupport (= 7.1.3.2) globalid (>= 0.3.6) - activemodel (7.1.3) - activesupport (= 7.1.3) - activerecord (7.1.3) - activemodel (= 7.1.3) - activesupport (= 7.1.3) + activemodel (7.1.3.2) + activesupport (= 7.1.3.2) + activerecord (7.1.3.2) + activemodel (= 7.1.3.2) + activesupport (= 7.1.3.2) timeout (>= 0.4.0) - activestorage (7.1.3) - actionpack (= 7.1.3) - activejob (= 7.1.3) - activerecord (= 7.1.3) - activesupport (= 7.1.3) + activestorage (7.1.3.2) + actionpack (= 7.1.3.2) + activejob (= 7.1.3.2) + activerecord (= 7.1.3.2) + activesupport (= 7.1.3.2) marcel (~> 1.0) - activesupport (7.1.3) + activesupport (7.1.3.2) base64 bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) @@ -79,8 +79,8 @@ GEM public_suffix (>= 2.0.2, < 6.0) ast (2.4.2) aws-eventstream (1.3.0) - aws-partitions (1.887.0) - aws-sdk-core (3.191.0) + aws-partitions (1.895.0) + aws-sdk-core (3.191.3) aws-eventstream (~> 1, >= 1.3.0) aws-partitions (~> 1, >= 1.651.0) aws-sigv4 (~> 1.8) @@ -122,21 +122,20 @@ GEM date (3.3.4) diff-lcs (1.5.1) docile (1.4.0) - dotenv (2.8.1) - dotenv-rails (2.8.1) - dotenv (= 2.8.1) - railties (>= 3.2) - drb (2.2.0) - ruby2_keywords + dotenv (3.1.0) + dotenv-rails (3.1.0) + dotenv (= 3.1.0) + railties (>= 6.1) + drb (2.2.1) erubi (1.12.0) ffi (1.16.3) globalid (1.2.1) activesupport (>= 6.1) - govuk-components (5.0.2) + govuk-components (5.2.1) html-attributes-utils (~> 1.0.0, >= 1.0.0) pagy (~> 6.0) - view_component (>= 3.9, < 3.10) - govuk_design_system_formbuilder (5.0.0) + view_component (>= 3.9, < 3.11) + govuk_design_system_formbuilder (5.2.0) actionview (>= 6.1) activemodel (>= 6.1) activesupport (>= 6.1) @@ -149,16 +148,17 @@ GEM i18n (1.14.1) concurrent-ruby (~> 1.0) io-console (0.7.2) - irb (1.11.1) + irb (1.11.2) rdoc reline (>= 0.4.2) jmespath (1.6.2) json (2.7.1) json-schema (4.1.1) addressable (>= 2.8) - jwt (2.7.1) + jwt (2.8.1) + base64 language_server-protocol (3.17.0.3) - listen (3.8.0) + listen (3.9.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) loofah (2.22.0) @@ -169,11 +169,11 @@ GEM net-imap net-pop net-smtp - marcel (1.0.2) + marcel (1.0.4) matrix (0.4.2) method_source (1.0.0) mini_mime (1.1.5) - minitest (5.22.0) + minitest (5.22.2) msgpack (1.7.2) mutex_m (0.2.0) net-imap (0.4.10) @@ -188,18 +188,16 @@ GEM nio4r (2.7.0) nokogiri (1.16.2-arm64-darwin) racc (~> 1.4) - nokogiri (1.16.2-x86_64-darwin) - racc (~> 1.4) nokogiri (1.16.2-x86_64-linux) racc (~> 1.4) notifications-ruby-client (5.4.0) jwt (>= 1.5, < 3) - pagy (6.4.3) + pagy (6.5.0) parallel (1.24.0) parser (3.3.0.5) ast (~> 2.4.1) racc - pg (1.5.4) + pg (1.5.6) pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) @@ -212,7 +210,7 @@ GEM puma (6.4.2) nio4r (~> 2.0) racc (1.7.3) - rack (3.0.9) + rack (3.0.9.1) rack-attack (6.7.0) rack (>= 1.0, < 4) rack-proxy (0.7.7) @@ -227,20 +225,20 @@ GEM rackup (2.1.0) rack (>= 3) webrick (~> 1.8) - rails (7.1.3) - actioncable (= 7.1.3) - actionmailbox (= 7.1.3) - actionmailer (= 7.1.3) - actionpack (= 7.1.3) - actiontext (= 7.1.3) - actionview (= 7.1.3) - activejob (= 7.1.3) - activemodel (= 7.1.3) - activerecord (= 7.1.3) - activestorage (= 7.1.3) - activesupport (= 7.1.3) + rails (7.1.3.2) + actioncable (= 7.1.3.2) + actionmailbox (= 7.1.3.2) + actionmailer (= 7.1.3.2) + actionpack (= 7.1.3.2) + actiontext (= 7.1.3.2) + actionview (= 7.1.3.2) + activejob (= 7.1.3.2) + activemodel (= 7.1.3.2) + activerecord (= 7.1.3.2) + activestorage (= 7.1.3.2) + activesupport (= 7.1.3.2) bundler (>= 1.15.0) - railties (= 7.1.3) + railties (= 7.1.3.2) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) actionview (>= 5.0.1.rc1) @@ -252,9 +250,9 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (7.1.3) - actionpack (= 7.1.3) - activesupport (= 7.1.3) + railties (7.1.3.2) + actionpack (= 7.1.3.2) + activesupport (= 7.1.3.2) irb rackup (>= 1.0.0) rake (>= 12.2) @@ -269,12 +267,12 @@ GEM ffi (~> 1.0) rdoc (6.6.2) psych (>= 4.0.0) - redis (5.0.8) + redis (5.1.0) redis-client (>= 0.17.0) - redis-client (0.19.1) + redis-client (0.21.0) connection_pool regexp_parser (2.9.0) - reline (0.4.2) + reline (0.4.3) io-console (~> 0.5) rexml (3.2.6) rspec-core (3.13.0) @@ -293,7 +291,7 @@ GEM rspec-expectations (~> 3.12) rspec-mocks (~> 3.12) rspec-support (~> 3.12) - rspec-support (3.13.0) + rspec-support (3.13.1) rubocop (1.60.2) json (~> 2.3) language_server-protocol (>= 3.17.0) @@ -332,7 +330,6 @@ GEM rubocop-capybara (~> 2.17) rubocop-factory_bot (~> 2.22) ruby-progressbar (1.13.0) - ruby2_keywords (0.0.5) rubyzip (2.3.2) sass (3.7.4) sass-listen (~> 4.0.0) @@ -344,7 +341,7 @@ GEM scss_lint-govuk (0.2.0) scss_lint securerandom (0.3.1) - selenium-webdriver (4.17.0) + selenium-webdriver (4.18.1) base64 (~> 0.2) rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) @@ -355,7 +352,7 @@ GEM sentry-ruby (~> 5.16.1) sentry-ruby (5.16.1) concurrent-ruby (~> 1.0, >= 1.0.2) - sidekiq (7.2.1) + sidekiq (7.2.2) concurrent-ruby (< 2) connection_pool (>= 2.3.0) rack (>= 2.2.4) @@ -367,13 +364,13 @@ GEM simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) stringio (3.1.0) - thor (1.3.0) + thor (1.3.1) timeout (0.4.1) tzinfo (2.0.6) concurrent-ruby (~> 1.0) uk_postcode (2.1.8) unicode-display_width (2.5.0) - view_component (3.9.0) + view_component (3.10.0) activesupport (>= 5.2.0, < 8.0) concurrent-ruby (~> 1.0) method_source (~> 1.0) @@ -398,7 +395,6 @@ GEM PLATFORMS arm64-darwin-22 - x86_64-darwin-23 x86_64-linux DEPENDENCIES