Merge branch 'main' into scriptrunner_save_protections

docs.openc3.com/docs/meta/licenses.md

@@ -12,11 +12,13 @@ Obviously, the actual license text applies, but here is a short summary:
 
 - The AGPL allows users to use the code however they want: For business, personal, etc., as long as they follow the other terms:
 
-  1. The software is provided as-is, no warranty
+  1. Users are anyone who could access the web-app. On the public internet, that is the whole world. On a private network, it is anyone with access to that network.
 
-  2. Users must be given access to all the source code and are also allowed to use it however they want under the same terms of the AGPLv3. This includes any modifications made, anything added, and all plugins.
+  2. The software is provided as-is, no warranty
 
-  3. For web applications (like COSMOS), a link must be provided to all of the source code.
+  3. Users must be given access to all the source code and are also allowed to use it however they want under the same terms of the AGPLv3. This includes any modifications made, anything added, and all plugins.
+
+  4. For web applications (like COSMOS), a link must be provided to all of the source code.
 
 - There are some key implications of the above:
 

openc3/lib/openc3/microservices/decom_microservice.rb

@@ -45,15 +45,17 @@ def initialize(*args)
     end
 
     def run
+      setup_microservice_topic()
       while true
         break if @cancel_thread
 
         begin
           OpenC3.in_span("read_topics") do
             Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
               break if @cancel_thread
-
-              if topic =~ /__DECOMINTERFACE/
+              if topic == @microservice_topic
+                microservice_cmd(topic, msg_id, msg_hash, redis)
+              elsif topic =~ /__DECOMINTERFACE/
                 if msg_hash.key?('inject_tlm')
                   handle_inject_tlm(msg_hash['inject_tlm'])
                   next

openc3/lib/openc3/microservices/log_microservice.rb

@@ -62,12 +62,17 @@ def initialize(name)
 
     def run
       setup_plws()
+      setup_microservice_topic()
       while true
         break if @cancel_thread
 
         Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
           break if @cancel_thread
-          log_data(topic, msg_id, msg_hash, redis)
+          if topic == @microservice_topic
+            microservice_cmd(topic, msg_id, msg_hash, redis)
+          else
+            log_data(topic, msg_id, msg_hash, redis)
+          end
           @count += 1
           @metric.set(name: 'log_total', value: @count, type: 'counter')
         end

openc3/lib/openc3/microservices/microservice.rb

@@ -116,6 +116,7 @@ def initialize(name, is_plugin: false)
       end
       @logger.info("Microservice initialized with config:\n#{@config}")
       @topics ||= []
+      @microservice_topic = "MICROSERVICE__#{@name}"
 
       # Get configuration for any targets
       @target_names = @config["target_names"]
@@ -210,5 +211,32 @@ def shutdown
       @logger.info("Shutting down microservice complete: #{@name}")
       @shutdown_complete = true
     end
+
+    def setup_microservice_topic
+      @topics.append(@microservice_topic)
+      Thread.current[:topic_offsets] ||= {}
+      topic_offsets = Thread.current[:topic_offsets]
+      topic_offsets[@microservice_topic] = "0-0" # Always get all available
+    end
+
+    # Returns if the command was handled
+    def microservice_cmd(topic, msg_id, msg_hash, redis)
+      command = msg_hash['command']
+      case command
+      when 'ADD_TOPICS'
+        topics = JSON.parse(msg_hash['topics'])
+        if topics and Array === topics
+          topics.each do |new_topic|
+            @topics << new_topic unless @topics.include?(new_topic)
+          end
+        else
+          raise "Invalid topics given to microservice_cmd: #{topics}"
+        end
+        Topic.trim_topic(topic, msg_id)
+        return true
+      end
+      Topic.trim_topic(topic, msg_id)
+      return false
+    end
   end
 end

openc3/lib/openc3/models/microservice_model.rb

@@ -43,6 +43,7 @@ class MicroserviceModel < Model
     attr_accessor :secrets
     attr_accessor :prefix
     attr_accessor :disable_erb
+    attr_accessor :ignore_changes
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
@@ -103,6 +104,7 @@ def initialize(
       secrets: [],
       prefix: nil,
       disable_erb: nil,
+      ignore_changes: nil,
       scope:
     )
       parts = name.split("__")
@@ -128,6 +130,7 @@ def initialize(
       @secrets = secrets
       @prefix = prefix
       @disable_erb = disable_erb
+      @ignore_changes = ignore_changes
       @bucket = Bucket.getClient()
     end
 
@@ -149,7 +152,8 @@ def as_json(*a)
         'needs_dependencies' => @needs_dependencies,
         'secrets' => @secrets.as_json(*a),
         'prefix' => @prefix,
-        'disable_erb' => @disable_erb
+        'disable_erb' => @disable_erb,
+        'ignore_changes' => @ignore_changes
       }
     end
 

openc3/lib/openc3/models/plugin_model.rb

@@ -342,7 +342,7 @@ def undeploy
         raise message
       end
     rescue Exception => error
-      Logger.error("Error undeploying plugin model #{@name} in scope #{@scope} due to: #{error}")
+      Logger.error("Error undeploying plugin model #{@name} in scope #{@scope} due to: #{error.formatted}")
     ensure
       # Double check everything is gone
       found = []

openc3/lib/openc3/models/target_model.rb

@@ -869,6 +869,42 @@ def dynamic_update(packets, cmd_or_tlm = :TELEMETRY, filename = "dynamic_tlm.txt
           )
         end
       end
+
+      # Inform microservices of new topics
+      # Need to tell loggers to log, and decom to decom
+      # We do this for no downtime
+      raw_topics = []
+      decom_topics = []
+      packets.each do |packet|
+        if cmd_or_tlm == :TELEMETRY
+          raw_topics << "#{@scope}__TELEMETRY__{#{@name}}__#{packet.packet_name.upcase}"
+          decom_topics << "#{@scope}__DECOM__{#{@name}}__#{packet.packet_name.upcase}"
+        else
+          raw_topics << "#{@scope}__COMMAND__{#{@name}}__#{packet.packet_name.upcase}"
+          decom_topics << "#{@scope}__DECOMCMD__{#{@name}}__#{packet.packet_name.upcase}"
+        end
+      end
+      if cmd_or_tlm == :TELEMETRY
+        Topic.write_topic("MICROSERVICE__#{@scope}__PACKETLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
+        add_topics_to_microservice("#{@scope}__PACKETLOG__#{@name}", raw_topics)
+        Topic.write_topic("MICROSERVICE__#{@scope}__DECOMLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => decom_topics.as_json.to_json})
+        add_topics_to_microservice("#{@scope}__DECOMLOG__#{@name}", decom_topics)
+        Topic.write_topic("MICROSERVICE__#{@scope}__DECOM__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
+        add_topics_to_microservice("#{@scope}__DECOM__#{@name}", raw_topics)
+      else
+        Topic.write_topic("MICROSERVICE__#{@scope}__COMMANDLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => raw_topics.as_json.to_json})
+        add_topics_to_microservice("#{@scope}__COMMANDLOG__#{@name}", raw_topics)
+        Topic.write_topic("MICROSERVICE__#{@scope}__DECOMCMDLOG__#{@name}", {'command' => 'ADD_TOPICS', 'topics' => decom_topics.as_json.to_json})
+        add_topics_to_microservice("#{@scope}__DECOMCMDLOG__#{@name}", decom_topics)
+      end
+    end
+
+    def add_topics_to_microservice(microservice_name, topics)
+      model = MicroserviceModel.get_model(name: microservice_name, scope: @scope)
+      model.topics.concat(topics)
+      model.topics.uniq!
+      model.ignore_changes = true # Don't restart the microservice right now
+      model.update
     end
 
     def deploy_commmandlog_microservice(gem_path, variables, topics, instance = nil, parent = nil)
@@ -1109,7 +1145,6 @@ def deploy_microservices(gem_path, variables, system)
       decom_command_topic_list = []
       packet_topic_list = []
       decom_topic_list = []
-      reduced_topic_list = []
       begin
         system.commands.packets(@name).each do |packet_name, packet|
           command_topic_list << "#{@scope}__COMMAND__{#{@name}}__#{packet_name}"
@@ -1122,9 +1157,6 @@ def deploy_microservices(gem_path, variables, system)
         system.telemetry.packets(@name).each do |packet_name, packet|
           packet_topic_list << "#{@scope}__TELEMETRY__{#{@name}}__#{packet_name}"
           decom_topic_list  << "#{@scope}__DECOM__{#{@name}}__#{packet_name}"
-          reduced_topic_list << "#{@scope}__REDUCED_MINUTE__{#{@name}}__#{packet_name}"
-          reduced_topic_list << "#{@scope}__REDUCED_HOUR__{#{@name}}__#{packet_name}"
-          reduced_topic_list << "#{@scope}__REDUCED_DAY__{#{@name}}__#{packet_name}"
         end
       rescue
         # No telemetry packets for this target

openc3/lib/openc3/operators/microservice_operator.rb

@@ -98,28 +98,30 @@ def update
           previous_parent = @previous_microservices[microservice_name]['parent']
           if @previous_microservices[microservice_name] != microservice_config
             # CHANGED
-            scope = microservice_name.split("__")[0]
-            Logger.info("Changed microservice detected: #{microservice_name}\nWas: #{@previous_microservices[microservice_name]}\nIs: #{microservice_config}", scope: scope)
-            if parent or previous_parent
-              if parent == previous_parent
-                # Same Parent - Respawn parent
-                @changed_microservices[parent] = @microservices[parent] if @microservices[parent] and @previous_microservices[parent]
-              elsif parent and previous_parent
-                # Parent changed - Respawn both parents
-                @changed_microservices[parent] = @microservices[parent] if @microservices[parent] and @previous_microservices[parent]
-                @changed_microservices[previous_parent] = @microservices[previous_parent] if @microservices[previous_parent] and @previous_microservices[previous_parent]
-              elsif parent
-                # Moved under a parent - Respawn parent and kill standalone
-                @changed_microservices[parent] = @microservices[parent] if @microservices[parent] and @previous_microservices[parent]
-                @removed_microservices[microservice_name] = microservice_config
-              else # previous_parent
-                # Moved to standalone - Respawn previous parent and make new
-                @changed_microservices[previous_parent] = @microservices[previous_parent] if @microservices[previous_parent] and @previous_microservices[previous_parent]
-                @new_microservices[microservice_name] = microservice_config
+            if not microservice_config['ignore_changes']
+              scope = microservice_name.split("__")[0]
+              Logger.info("Changed microservice detected: #{microservice_name}\nWas: #{@previous_microservices[microservice_name]}\nIs: #{microservice_config}", scope: scope)
+              if parent or previous_parent
+                if parent == previous_parent
+                  # Same Parent - Respawn parent
+                  @changed_microservices[parent] = @microservices[parent] if @microservices[parent] and @previous_microservices[parent]
+                elsif parent and previous_parent
+                  # Parent changed - Respawn both parents
+                  @changed_microservices[parent] = @microservices[parent] if @microservices[parent] and @previous_microservices[parent]
+                  @changed_microservices[previous_parent] = @microservices[previous_parent] if @microservices[previous_parent] and @previous_microservices[previous_parent]
+                elsif parent
+                  # Moved under a parent - Respawn parent and kill standalone
+                  @changed_microservices[parent] = @microservices[parent] if @microservices[parent] and @previous_microservices[parent]
+                  @removed_microservices[microservice_name] = microservice_config
+                else # previous_parent
+                  # Moved to standalone - Respawn previous parent and make new
+                  @changed_microservices[previous_parent] = @microservices[previous_parent] if @microservices[previous_parent] and @previous_microservices[previous_parent]
+                  @new_microservices[microservice_name] = microservice_config
+                end
+              else
+                # Respawn regular microservice
+                @changed_microservices[microservice_name] = microservice_config
               end
-            else
-              # Respawn regular microservice
-              @changed_microservices[microservice_name] = microservice_config
             end
           end
         else

openc3/lib/openc3/packets/commands.rb

@@ -303,6 +303,10 @@ def all
       @config.commands
     end
 
+    def dynamic_add_packet(packet, affect_ids: false)
+      @config.dynamic_add_packet(packet, :COMMAND, affect_ids: affect_ids)
+    end
+
     protected
 
     def set_parameters(command, params, range_checking)

openc3/lib/openc3/packets/packet_config.rb

@@ -315,30 +315,60 @@ def finish_packet
           hash = @cmd_id_value_hash[@current_packet.target_name]
           hash = {} unless hash
           @cmd_id_value_hash[@current_packet.target_name] = hash
-          update_id_value_hash(hash)
+          update_id_value_hash(@current_packet, hash)
         else
           @telemetry[@current_packet.target_name][@current_packet.packet_name] = @current_packet
           hash = @tlm_id_value_hash[@current_packet.target_name]
           hash = {} unless hash
           @tlm_id_value_hash[@current_packet.target_name] = hash
-          update_id_value_hash(hash)
+          update_id_value_hash(@current_packet, hash)
         end
         @current_packet = nil
         @current_item = nil
       end
     end
 
+    def dynamic_add_packet(packet, cmd_or_tlm = :TELEMETRY, affect_ids: false)
+      if cmd_or_tlm == :COMMAND
+        @commands[packet.target_name][packet.packet_name] = packet
+
+        if affect_ids
+          hash = @cmd_id_value_hash[packet.target_name]
+          hash = {} unless hash
+          @cmd_id_value_hash[packet.target_name] = hash
+          update_id_value_hash(packet, hash)
+        end
+      else
+        @telemetry[packet.target_name][packet.packet_name] = packet
+
+        # Update latest_data lookup for telemetry
+        packet.sorted_items.each do |item|
+          target_latest_data = @latest_data[packet.target_name]
+          target_latest_data[item.name] ||= []
+          latest_data_packets = target_latest_data[item.name]
+          latest_data_packets << packet unless latest_data_packets.include?(packet)
+        end
+
+        if affect_ids
+          hash = @tlm_id_value_hash[packet.target_name]
+          hash = {} unless hash
+          @tlm_id_value_hash[packet.target_name] = hash
+          update_id_value_hash(packet, hash)
+        end
+      end
+    end
+
     protected
 
-    def update_id_value_hash(hash)
-      if @current_packet.id_items.length > 0
+    def update_id_value_hash(packet, hash)
+      if packet.id_items.length > 0
         key = []
-        @current_packet.id_items.each do |item|
+        packet.id_items.each do |item|
           key << item.id_value
         end
-        hash[key] = @current_packet
+        hash[key] = packet
       else
-        hash['CATCHALL'.freeze] = @current_packet
+        hash['CATCHALL'.freeze] = packet
       end
     end
 

openc3/lib/openc3/packets/telemetry.rb

@@ -442,5 +442,9 @@ def all_item_strings(include_hidden = false, splash = nil)
     def all
       @config.telemetry
     end
+
+    def dynamic_add_packet(packet, affect_ids: false)
+      @config.dynamic_add_packet(packet, :TELEMETRY, affect_ids: affect_ids)
+    end
   end # class Telemetry
 end

openc3/lib/openc3/system/system.rb

@@ -60,6 +60,9 @@ class System
     # The current limits set
     @@limits_set = nil
 
+    # Callbacks to call once @@instance is created
+    @@post_instance_callbacks = []
+
     # @return [Symbol] The current limits_set of the system returned from Redis
     def self.limits_set
       unless @@limits_set
@@ -72,6 +75,14 @@ def self.limits_set=(value)
       @@limits_set = value.to_s.intern
     end
 
+    def self.add_post_instance_callback(callback)
+      if @@instance
+        callback.call()
+      else
+        @@post_instance_callbacks << callback
+      end
+    end
+
     def self.setup_targets(target_names, base_dir, scope:)
       # Nothing to do if there are no targets
       return if target_names.nil? or target_names.length == 0
@@ -121,11 +132,30 @@ def self.instance(target_names = nil, target_config_dir = nil)
       raise "System.instance parameters are required on first call" unless target_names and target_config_dir
 
       @@instance_mutex.synchronize do
+        return @@instance if @@instance
         @@instance ||= self.new(target_names, target_config_dir)
+        @@post_instance_callbacks.each do |callback|
+          callback.call
+        end
         return @@instance
       end
     end
 
+    # Dynamically add packets to the system instance
+    #
+    # @param dynamic_packets [Array of packets]
+    # @param cmd_or_tlm [Symbol] :COMMAND or :TELEMETRY
+    # @param affect_ids [Boolean] Whether to affect packet id lookup or not
+    def self.dynamic_update(dynamic_packets, cmd_or_tlm = :TELEMETRY, affect_ids: false)
+      dynamic_packets.each do |packet|
+        if cmd_or_tlm == :TELEMETRY
+          @@instance.telemetry.dynamic_add_packet(packet, affect_ids: affect_ids)
+        else
+          @@instance.commands.dynamic_add_packet(packet, affect_ids: affect_ids)
+        end
+      end
+    end
+
     # Create a new System object.
     #
     # @param target_names [Array of target names]