Fix SNS throttling error finding SNS topics

.github/workflows/rspec.yml

@@ -7,20 +7,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3.5.3
 
       - name: Run tests
         run: |
-          cd  script
-          docker-compose up -d
-          docker exec testrunner bash -c "cd src && bundle install && bundle exec rspec && exit"
+          docker-compose up -d --build
+          docker exec testrunner bash -c "bundle exec rspec spec && exit"
 
       - name: Code Coverage
-        uses: paambaati/codeclimate-action@v2.7.5
+        uses: paambaati/codeclimate-action@v5.0.0
         env:
           CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
         with:
-          prefix: '/src'
+          verifyDownload: true
           coverageLocations: |
             ${{github.workspace}}/coverage/.resultset.json:simplecov
 

.gitignore

@@ -15,3 +15,6 @@
 process_file.txt
 spec_run.txt
 /utilities/plot_results
+
+tags
+*.tags

Dockerfile

@@ -0,0 +1,15 @@
+FROM ruby:2.4-alpine3.8
+
+RUN apk add --update ca-certificates bash && update-ca-certificates && rm -rf /var/cache/apk/*
+RUN apk update && apk add --no-cache build-base
+
+RUN set -ex && apk add --no-cache --virtual .gem-builddeps ruby-dev build-base libressl-dev
+
+ENV APP_HOME /src
+RUN mkdir -p $APP_HOME
+WORKDIR $APP_HOME
+
+COPY . $APP_HOME
+RUN bundle install
+RUN apk del .gem-builddeps
+

EVENTQ_VERSION

@@ -1 +1 @@
-3.3.0
+3.3.1

Gemfile

@@ -1,14 +1,19 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in eventq.gemspec
 gemspec
 
-
 gem 'json', '2.1.0'
 gem 'redlock'
 
 platforms :ruby do
+  gem 'bunny'
   gem 'oj', '3.6.10'
   gem 'openssl', '2.1.1'
-  gem 'bunny'
+end
+
+group :development, :test do
+  gem 'pry-byebug', '< 3.10' # 3.10.0 requires Ruby >= 2.7
 end

script/docker-compose.yml → docker-compose.yml

@@ -7,20 +7,20 @@ services:
       dockerfile: Dockerfile
     image: eventq
     container_name: testrunner
-    command: bash -c "./scripts/container_loop.sh"
+    command: bash -c "/scripts/container_loop.sh"
     depends_on:
       - redis
       - rabbitmq
       - localstack
     volumes:
-      - ./container_loop.sh:/scripts/container_loop.sh
-      - ../:/src
+      - ./script/container_loop.sh:/scripts/container_loop.sh
+      - ./:/src
     environment:
       - AWS_ACCESS_KEY_ID=mock_id
-      - AWS_SECRET_ACCESS_KEY=mock_password
       - AWS_REGION=eu-west-1
-      - AWS_SQS_ENDPOINT=http://localstack:4566
+      - AWS_SECRET_ACCESS_KEY=mock_password
       - AWS_SNS_ENDPOINT=http://localstack:4566
+      - AWS_SQS_ENDPOINT=http://localstack:4566
 #    env_file:
 #      - ../.aws.env
 
@@ -31,8 +31,6 @@ services:
   redis:
     image: redis:alpine
     container_name: eventq_redis
-    ports:
-      - "6379:6379"
 
   localstack:
     # changes to multi region support after this version break tests

eventq.gemspec

@@ -1,31 +1,34 @@
-# coding: utf-8
+# frozen_string_literal: true
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-version = File.read(File.expand_path("EVENTQ_VERSION", __dir__)).strip
+version = File.read(File.expand_path('EVENTQ_VERSION', __dir__)).strip
 
 Gem::Specification.new do |spec|
-  spec.name          = "eventq"
+  spec.name          = 'eventq'
   spec.version       = version
-  spec.authors       = ["SageOne"]
-  spec.email         = ["sageone@sage.com"]
+  spec.authors       = ['SageOne']
+  spec.email         = ['sageone@sage.com']
 
   spec.description = spec.summary = 'EventQ is a pub/sub system that uses async notifications and message queues'
-  spec.homepage      = "https://github.com/sage/eventq"
-  spec.license       = "MIT"
-  spec.files         = ["README.md"] + Dir.glob("{bin,lib}/**/**/**")
-  spec.require_paths = ["lib"]
+  spec.homepage      = 'https://github.com/sage/eventq'
+  spec.license       = 'MIT'
+  spec.files         = ['README.md'] + Dir.glob('{bin,lib}/**/**/**')
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = '>= 2.4'
 
   spec.add_development_dependency 'activesupport', '~> 4'
-  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'byebug', '~> 11.0'
   spec.add_development_dependency 'pry-byebug', '~> 3.9'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'shoulda-matchers'
   spec.add_development_dependency 'simplecov', '< 0.18.0'
 
-  spec.add_dependency 'aws-sdk-sqs', '~> 1'
   spec.add_dependency 'aws-sdk-sns', '~> 1'
+  spec.add_dependency 'aws-sdk-sqs', '~> 1'
   spec.add_dependency 'bunny'
   spec.add_dependency 'class_kit'
   spec.add_dependency 'concurrent-ruby'

lib/eventq.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 require 'redlock'
 require 'class_kit'
@@ -22,6 +24,3 @@
 require_relative 'eventq/eventq_base/signature_providers'
 require_relative 'eventq/eventq_base/exceptions'
 require_relative 'eventq/queue_worker'
-
-
-

lib/eventq/eventq_aws/sns.rb

@@ -8,10 +8,11 @@ module Amazon
     class SNS
       @@topic_arns = Concurrent::Hash.new
 
-      attr_reader :sns
+      attr_reader :sns, :sns_resource
 
-      def initialize(client)
+      def initialize(client, resource = nil)
         @sns = client
+        @sns_resource = resource || Aws::SNS::Resource.new(client: client)
       end
 
       # Create a TopicArn. if one already exists, it will return a pre-existing ARN from the cache.
@@ -67,26 +68,10 @@ def aws_safe_name(name)
 
       # Finds the given topic, or returns nil if the topic could not be found
       #
-      # @note Responses to list_topics can be paged, so to check *all* topics
-      # we'll need to request each page of topics using response.next_token for
-      # until the response no longer contains a next_token. Requests to
-      # list_topics are throttled to 30 TPS, so in the future we may need to
-      # handle this if it becomes a problem.
-      #
       # @param topic_name [String] the name of the topic to find
       # @return [String]
       def find_topic(topic_name)
-        response = sns.list_topics
-        topics = response.topics
-        arn = topics.detect { |topic| topic.topic_arn.end_with?(":#{topic_name}") }&.topic_arn
-
-        while arn.nil? && response.next_token
-          response = sns.list_topics(next_token: response.next_token)
-          topics = response.topics
-          arn = topics.detect { |topic| topic.topic_arn.end_with?(":#{topic_name}") }&.topic_arn
-        end
-
-        arn
+        sns_resource.topics.detect { |topic| topic.arn.end_with?(":#{topic_name}") }&.arn
       end
     end
   end

spec/spec_helper.rb

@@ -7,7 +7,6 @@
 SimpleCov.start do
   add_filter ['spec/']
 end
-puts 'required simplecov'
 require 'shoulda-matchers'
 
 Shoulda::Matchers.configure do |config|
@@ -21,11 +20,6 @@
 require_relative '../lib/eventq/aws'
 
 RSpec.configure do |config|
-
-  config.before(:each) do
-
-  end
-
   config.expect_with :rspec do |expectations|
     expectations.include_chain_clauses_in_custom_matcher_descriptions = true
   end