Refined address processing

docker-compose.yml

@@ -19,7 +19,7 @@ services:
   ######  dynamic configuration  ###########
   generate-raw-addresses:
     image: techbureau/catapult-tools-private:gcc-master-ac5add6c2c
-    command: bash -c "/bin-mount/generate-raw-addresses-if-needed 50 /addresses/raw-addresses.txt /addresses/addresses.yaml"
+    command: bash -c "/bin-mount/generate-raw-addresses-if-needed 53 /addresses/raw-addresses.txt /addresses/addresses.yaml"
     volumes:
     - ./bin/bash:/bin-mount
     - ./build/generated-addresses:/addresses:rw
@@ -29,7 +29,8 @@ services:
     build:
       context: ./ruby
       dockerfile: Dockerfile-dev
-    command: bash -c "/bin-mount/wait /addresses/raw-addresses.txt && /usr/app/bin/store-addresses-if-needed.rb /addresses/raw-addresses.txt /addresses/addresses.yaml"
+    command: bash -c "sleep infinity"
+#    command: bash -c "/bin-mount/wait /addresses/raw-addresses.txt && /usr/app/bin/store-addresses-if-needed.rb /addresses/raw-addresses.txt /addresses/addresses.yaml 53"
     volumes:
     - ./bin/bash:/bin-mount
     - ./ruby:/usr/app
@@ -39,8 +40,8 @@ services:
     build:
       context: ./ruby
       dockerfile: Dockerfile-dev
-#    command: bash -c "sleep infinity"
-    command: bash -c "/bin-mount/wait /addresses/addresses.yaml && /usr/app/bin/generate-and-write-configurations.rb /addresses/addresses.yaml /config-build /nemesis && touch /state/configs-generated"
+    command: bash -c "sleep infinity"
+#    command: bash -c "/bin-mount/wait /addresses/addresses.yaml && /usr/app/bin/generate-and-write-configurations.rb /addresses/addresses.yaml /config-build /nemesis && touch /state/configs-generated"
     volumes:
     - ./bin/bash:/bin-mount
     - ./ruby:/usr/app

ruby/Gemfile.lock

@@ -2,6 +2,7 @@ PATH
   remote: .
   specs:
     catapult-bootstrap (0.1.0)
+      frontkick (~> 0.5, >= 0.5.7)
       mustache (~> 1.1, >= 1.1.1)
 
 GEM
@@ -10,6 +11,7 @@ GEM
     byebug (11.1.1)
     coderay (1.1.2)
     diff-lcs (1.3)
+    frontkick (0.5.7)
     method_source (1.0.0)
     mustache (1.1.1)
     pry (0.13.0)

ruby/bin/store-addresses-if-needed.rb

@@ -13,10 +13,17 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 require_relative('../lib/catapult.rb')
+unless ARGV.size == 3
+  STDERR << "Usage #{$0} INPUT_FILE_PATH OUTPUT_FILE_PATH TOTAL_ADDRESS\n"
+  exit 1
+end
+
 input_file_path = ARGV[0]
 output_file_path = ARGV[1]     
+total_addresses  = ARGV[2].to_i
+
 fail "No file exists at path '#{input_file_path}'" unless File.file?(input_file_path)
 unless File.file?(output_file_path)
-  ruby_object = Catapult::Bootstrap::Addresses.parse(input_file_path)
-  File.open(output_file_path, 'w') { |f| f << YAML.dump(ruby_object) }
+  ruby_object = Catapult::Bootstrap::Addresses.parse(input_file_path, total_addresses, output_form: :hash_for_yaml)
+  File.open(output_file_path, 'w') { |f| f << ::YAML.dump(ruby_object) }
 end

ruby/catapult-bootstrap.gemspec

@@ -11,6 +11,7 @@ Gem::Specification.new do |s|
   s.license     = "Apache-2.0."
   s.required_ruby_version = ">= 2.5"
   s.add_dependency 'mustache', '~> 1.1', '>= 1.1.1'
+  s.add_dependency 'frontkick', '~> 0.5', '>= 0.5.7'
   s.add_development_dependency 'rspec', '~> 3.6', '>= 3.6.0'
 
   s.files         = `find *`.split("\n").uniq.sort.select { |f| !f.empty? }.reject { |f| f =~ /gem$/ or f =~ /gemspec$/ }

ruby/lib/catapult/bootstrap.rb

@@ -18,6 +18,7 @@ module Bootstrap
     # global must go first
     require_relative('bootstrap/addresses')
     require_relative('bootstrap/config')
+    require_relative('bootstrap/tools')
 
     BASE_CONFIG_SOURCE_DIR  = File.expand_path('../../catapult-templates', File.dirname(__FILE__))
 

ruby/lib/catapult/bootstrap/addresses.rb

@@ -12,82 +12,58 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 module Catapult::Bootstrap
-  module Addresses
-    SECTION_SIZES = {
-      Global::ParseKey.peer_nodes                   => 5,
-      Global::ParseKey.api_nodes                    => 2,
-      Global::ParseKey.rest_gateways                => 2,
-      Global::ParseKey.nemesis_addresses_harvesting => 3,
-      Global::ParseKey.nemesis_generation_hash      => 1,
-      Global::ParseKey.nemesis_signer_private_key   => 1
-      # Global::ParseKey.nemesis_addresses => * # dyanmically calculated using left over address
+  class Addresses
+    require_relative('addresses/parse')
+    include Global
+
+    DEFAULT_SECTION_SIZES = {
+      ParseKey.peer_nodes                   => 5,
+      ParseKey.api_nodes                    => 2,
+      ParseKey.rest_gateways                => 2,
+      ParseKey.nemesis_addresses_harvesting => 3,
+      ParseKey.nemesis_generation_hash      => 1,
+      ParseKey.nemesis_signer_private_key   => 1
     }
 
-    # returns ruby object
-    def self.parse(input_file_path)
-      parsed_flat_form = parse_into_flat_form(input_file_path)
-      break_into_sections(parsed_flat_form)
+    def initialize(input_file_path, address_total)
+      @input_file_path  = input_file_path
+      @address_total    = address_total
+    end
+
+    def self.parse(input_file_path, address_total, output_form: nil)
+      new(input_file_path, address_total).parse(output_form: output_form)
+    end
+    def parse(output_form: nil)
+      Parse.parse(self.raw_address_info, self.section_sizes, break_into_sections: true, output_form: output_form)
     end
 
-    private
+    protected
+
+    attr_reader :input_file_path, :address_total
 
-    def self.parse_into_flat_form(input_file_path)
-      parsed_form = []
-      next_state   = :private # states can be :private, :public, :address
-      next_element = {}
-      File.open(input_file_path).read.each_line do |line|
-        next unless line =~ Regexp.new("#{next_state}")
-        add_to_element!(next_element, next_state, line)
-        case next_state
-        when :private 
-        next_state = :public
-        when :public
-          next_state = :address
-        when :address
-          parsed_form << next_element
-          next_element = {}
-          next_state = :private
-        end
-      end
-      parsed_form
+    def raw_address_info
+      @raw_address_info ||= ::File.open(self.input_file_path).read
     end
 
+    def section_sizes
+      @section_sizes ||= ret_section_sizes
+    end
 
-    def self.break_into_sections(parsed_flat_form)
-      num_nemesis_addresses = parsed_flat_form.size - SECTION_SIZES.values.inject(0, :+) - 2
-      unless num_nemesis_addresses > 0
-        fail "Not enough addresses"
-      end
-      parsed_form = {}
-      index       = 0
-      SECTION_SIZES.each_pair do |component_type, size|
-        parsed_form[component_type.to_s] = parsed_flat_form[index..index+size-1]
-        index = index+size
-      end
-      parsed_form['nemesis_addresses'] = parsed_flat_form[index..index+num_nemesis_addresses-1]
-      parsed_form
+    def num_harvesting_keys
+      @num_harvesting_keys ||= 
+        DEFAULT_SECTION_SIZES[ParseKey.nemesis_addresses_harvesting] || 
+        fail("Unexpected that ParseKey.nemesis_addresses_harvesting is nil")
     end
 
-    def self.add_to_element!(element, state, line)
-      value = 
-        case state
-        when :private 
-        value(state, line, /private key: ([0-9A-Z]+)/)
-        when :public
-          value(state, line, /public key: ([0-9A-Z]+)/)
-        when :address
-          value(state, line, Regexp.new("address \\(#{Global.catapult_nework_identifier}\\): ([0-9A-Z]+)"))
-        end
-      element.merge!(state.to_s => value)
-    end
-
-    def self.value(state, line, regexp)
-      unless line =~ regexp
-        fail "Cannot find #{state} data"
-      end
-      $1
+    private
+
+    def ret_section_sizes
+      # Rules are that nemesis_addresses_harvesting_vrf has same size as nemesis_addresses_harvesting and
+      # remaining addresses used in nemesis_addresses
+      ret = DEFAULT_SECTION_SIZES.merge(ParseKey.nemesis_addresses_harvesting_vrf => self.num_harvesting_keys)
+      num_nemesis_addresses = self.address_total - ret.values.sum
+      fail "Not enough total addresses to generate nemesis addresses" if num_nemesis_addresses < 1
+      ret.merge(ParseKey.nemesis_addresses => num_nemesis_addresses)
     end
   end
 end
-
-

ruby/lib/catapult/bootstrap/addresses/parse.rb

@@ -0,0 +1,114 @@
+#    Copyright 2018 Tech Bureau, Corp
+# 
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+# 
+#        http://www.apache.org/licenses/LICENSE-2.0
+# 
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+module Catapult::Bootstrap
+  class Addresses
+    class Parse
+      DEFAULT_OUTPUT_FORM = :element
+      def initialize(raw_address_info, section_sizes, break_into_sections: false, output_form: nil)
+        @raw_address_info    = raw_address_info
+        @section_sizes       = section_sizes
+        @break_into_sections = break_into_sections
+        @output_form         = output_form || DEFAULT_OUTPUT_FORM
+      end
+
+      ADDRESS_TYPES = [:private, :public, :address]
+      Element = Struct.new(*ADDRESS_TYPES)
+
+      def self.parse(raw_address_info, section_sizes, break_into_sections: false, output_form: nil)
+        new(raw_address_info, section_sizes, break_into_sections: break_into_sections, output_form: output_form).parse
+      end
+      def parse
+        self.should_break_into_sections? ? break_into_sections(self.parsed_flat_form) : self.parsed_flat_form
+      end
+
+      protected
+
+      attr_reader :raw_address_info, :section_sizes, :output_form
+
+      def should_break_into_sections?
+        @break_into_sections
+      end
+
+      def parsed_flat_form
+        @parsed_flat_form ||= parse_into_flat_form
+      end
+
+      private
+
+      def parse_into_flat_form
+        parsed_form = []
+        next_state   = :private # states can be :private, :public, :address
+        next_element = {}
+        self.raw_address_info.each_line do |line|
+          next unless line =~ ::Regexp.new("#{next_state}")
+          add_to_element!(next_element, next_state, line)
+          case next_state
+          when :private 
+            next_state = :public
+          when :public
+            next_state = :address
+          when :address
+            parsed_form << next_element
+            next_element = {}
+            next_state = :private
+          end
+        end
+        render_in_output_form(parsed_form)
+      end
+
+      def break_into_sections(parsed_flat_form)
+        parsed_form = {}
+        index       = 0
+        self.section_sizes.each_pair do |component_type, size|
+          parsed_form[component_type.to_s] = parsed_flat_form[index..index+size-1]
+          index += size
+        end
+        parsed_form
+      end
+
+      def add_to_element!(element, state, line)
+        value = 
+          case state
+          when :private 
+            value(state, line, /private key: ([0-9A-Z]+)/)
+          when :public
+            value(state, line, /public key: ([0-9A-Z]+)/)
+          when :address
+            value(state, line, Regexp.new("address \\(#{Global.catapult_nework_identifier}\\): ([0-9A-Z]+)"))
+          end
+        element.merge!(state => value)
+      end
+
+      def value(state, line, regexp)
+        unless line =~ regexp
+          fail "Cannot find #{state} data"
+        end
+        $1
+      end
+
+      def render_in_output_form(parsed_form)
+        case self.output_form
+        when :element
+          parsed_form.map { |hash| Element.new(hash[:private], hash[:public], hash[:address]) }
+        when :hash_for_yaml
+          parsed_form.map { |hash| hash.inject({}) { |h, (k, v)| h.merge(k.to_s => v) } }
+        else
+          fail "Illegal output_form '#{self.output_form}'"
+        end
+      end
+
+    end
+  end
+end
+

ruby/lib/catapult/bootstrap/global.rb

@@ -44,26 +44,19 @@ def self.catapult_nework_identifier
     end
 
     module ParseKey
-      def self.peer_nodes
-        :peer_nodes
-      end
-      def self.api_nodes
-        :api_nodes
-      end
-      def self.rest_gateways
+      ARRAY = [
+        :nemesis_addresses,
+        :nemesis_addresses_harvesting,
+        :nemesis_addresses_harvesting_vrf,
+        :nemesis_generation_hash,
+        :nemesis_signer_private_key,
+        # TODO: below wil be removed
+        :peer_nodes,
+        :api_nodes,
         :rest_gateways
-      end
-      def self.nemesis_addresses
-        :nemesis_addresses
-      end
-      def self.nemesis_addresses_harvesting
-        :nemesis_addresses_harvesting
-      end
-      def self.nemesis_generation_hash
-         :nemesis_generation_hash
-      end
-      def self.nemesis_signer_private_key
-        :nemesis_signer_private_key
+      ]
+      class << self
+        ARRAY.each { |el| define_method(el, lambda { el }) }
       end
     end