Merge pull request #1048 from OpenC3/dynamic_cmd_tlm

Dynamic target packet definition creation support.

docs.openc3.com/package.json

@@ -16,18 +16,18 @@
     "write-heading-ids": "docusaurus write-heading-ids"
   },
   "dependencies": {
-    "@docusaurus/core": "3.0.0",
-    "@docusaurus/plugin-client-redirects": "3.0.0",
-    "@docusaurus/preset-classic": "3.0.0",
+    "@docusaurus/core": "3.1.0",
+    "@docusaurus/plugin-client-redirects": "3.1.0",
+    "@docusaurus/preset-classic": "3.1.0",
     "@mdx-js/react": "3.0.0",
-    "docusaurus-lunr-search": "3.3.0",
-    "prism-react-renderer": "2.1.0",
+    "docusaurus-lunr-search": "3.3.2",
+    "prism-react-renderer": "2.3.1",
     "react": "18.2.0",
     "react-dom": "18.2.0"
   },
   "devDependencies": {
-    "@docusaurus/module-type-aliases": "3.0.0",
-    "@docusaurus/types": "3.0.0"
+    "@docusaurus/module-type-aliases": "3.1.0",
+    "@docusaurus/types": "3.1.0"
   },
   "browserslist": {
     "production": [

openc3-cosmos-cmd-tlm-api/app/controllers/users_controller.rb

@@ -21,5 +21,12 @@
   require 'openc3-enterprise/controllers/users_controller'
 rescue LoadError
   class UsersController < ApplicationController
+    def active()
+      render :json => [], :status => 200
+    end
+
+    def logout()
+      head :ok
+    end
   end
 end

openc3-cosmos-init/Dockerfile

@@ -88,11 +88,6 @@ FROM openc3-frontend-tmp AS openc3-tmp3
 COPY ./plugins/packages/openc3-cosmos-tool-packetviewer/ packages/openc3-cosmos-tool-packetviewer/
 RUN ["/openc3/plugins/docker-package-build.sh", "openc3-cosmos-tool-packetviewer"]
 
-# Build docs tool
-COPY --from=docs . packages/openc3-cosmos-tool-docs/
-RUN cd packages/openc3-cosmos-tool-docs/scripts && ruby generate_docs_from_yaml.rb PLUGIN && yarn
-RUN ["/openc3/plugins/docker-package-build.sh", "openc3-cosmos-tool-docs"]
-
 # Build tlmgrapher tool
 FROM openc3-frontend-tmp AS openc3-tmp4
 COPY ./plugins/packages/openc3-cosmos-tool-tlmgrapher/ packages/openc3-cosmos-tool-tlmgrapher/
@@ -110,13 +105,23 @@ RUN ["/openc3/plugins/docker-package-build.sh", "openc3-cosmos-tool-scriptrunner
 COPY ./plugins/packages/openc3-cosmos-demo/ packages/openc3-cosmos-demo/
 RUN ["/openc3/plugins/docker-package-build.sh", "openc3-cosmos-demo"]
 
+FROM ${OPENC3_REGISTRY}/${OPENC3_NAMESPACE}/${OPENC3_BASE_IMAGE}:${OPENC3_TAG} as base
+FROM openc3-frontend-tmp AS openc3-tmp5
+
+# Build docs tool
+COPY --from=base /openc3 /openc3/
+COPY --from=docs . packages/openc3-cosmos-tool-docs/
+RUN cd packages/openc3-cosmos-tool-docs/scripts && ruby generate_docs_from_yaml.rb PLUGIN && yarn
+RUN ["/openc3/plugins/docker-package-build.sh", "openc3-cosmos-tool-docs"]
+
 FROM ${OPENC3_REGISTRY}/${OPENC3_NAMESPACE}/${OPENC3_BASE_IMAGE}:${OPENC3_TAG}
 
 COPY --from=openc3-tmp1 --chown=${IMAGE_USER}:${IMAGE_GROUP} /openc3/plugins/packages/openc3-cosmos-ace-diff/ /openc3/plugins/packages/openc3-cosmos-ace-diff/
 COPY --from=openc3-tmp1 --chown=${IMAGE_USER}:${IMAGE_GROUP} /openc3/plugins/gems/* /openc3/plugins/gems/
 COPY --from=openc3-tmp2 --chown=${IMAGE_USER}:${IMAGE_GROUP} /openc3/plugins/gems/* /openc3/plugins/gems/
 COPY --from=openc3-tmp3 --chown=${IMAGE_USER}:${IMAGE_GROUP} /openc3/plugins/gems/* /openc3/plugins/gems/
 COPY --from=openc3-tmp4 --chown=${IMAGE_USER}:${IMAGE_GROUP} /openc3/plugins/gems/* /openc3/plugins/gems/
+COPY --from=openc3-tmp5 --chown=${IMAGE_USER}:${IMAGE_GROUP} /openc3/plugins/gems/* /openc3/plugins/gems/
 COPY --from=openc3-frontend-tmp --chown=${IMAGE_USER}:${IMAGE_GROUP} /openc3/plugins/packages/openc3-tool-common/ /openc3/plugins/packages/openc3-tool-common/
 COPY --from=openc3-frontend-tmp --chown=${IMAGE_USER}:${IMAGE_GROUP} /openc3/plugins/yarn.lock /openc3/plugins/yarn.lock
 COPY --from=openc3-frontend-tmp --chown=${IMAGE_USER}:${IMAGE_GROUP} /openc3/plugins/openc3-tool-base/yarn.lock /openc3/plugins/yarn-tool-base.lock

openc3-node/Dockerfile

@@ -2,7 +2,7 @@ ARG OPENC3_REGISTRY=docker.io
 ARG OPENC3_NAMESPACE=openc3inc
 ARG OPENC3_TAG=latest
 
-FROM ${OPENC3_REGISTRY}/${OPENC3_NAMESPACE}/openc3-base:${OPENC3_TAG}
+FROM ${OPENC3_REGISTRY}/${OPENC3_NAMESPACE}/openc3-ruby:${OPENC3_TAG}
 
 USER root
 

openc3-node/Dockerfile-ubi

@@ -2,7 +2,7 @@ ARG OPENC3_REGISTRY=docker.io
 ARG OPENC3_NAMESPACE=openc3inc
 ARG OPENC3_TAG=latest
 
-FROM ${OPENC3_REGISTRY}/${OPENC3_NAMESPACE}/openc3-base-ubi:${OPENC3_TAG}
+FROM ${OPENC3_REGISTRY}/${OPENC3_NAMESPACE}/openc3-ruby-ubi:${OPENC3_TAG}
 
 ENV NODEJS_VERSION=18
 

openc3/bin/openc3cli

@@ -350,7 +350,10 @@ def update_plugin(plugin_file_path, plugin_name, variables: nil, plugin_txt_line
     end
   rescue => e
     puts e.formatted
-    plugin_model.restore if plugin_model.destroyed?
+    if plugin_model.destroyed?
+      plugin_model.restore
+      # Local mode files should still be good because restore will now reuse the old name
+    end
     raise e
   end
 end

openc3/lib/openc3/accessors/http_accessor.rb

@@ -121,6 +121,34 @@ def write_item(item, value, buffer)
       return value
     end
 
+    def read_items(items, buffer)
+      result = {}
+      body_items = []
+      items.each do |item|
+        if item.name[0..4] == 'HTTP_'
+          result[item.name] = read_item(item, buffer)
+        else
+          body_items << item
+        end
+      end
+      body_result = @body_accessor.read_items(body_items, buffer)
+      result.merge!(body_result) # Merge Body accessor read items with HTTP_ items
+      return result
+    end
+
+    def write_items(items, values, buffer)
+      body_items = []
+      items.each_with_index do |item, index|
+        if item.name[0..4] == 'HTTP_'
+          write_item(item, values[index], buffer)
+        else
+          body_items << item
+        end
+      end
+      @body_accessor.write_items(body_items, values, buffer)
+      return values
+    end
+
     # If this is set it will enforce that buffer data is encoded
     # in a specific encoding
     def enforce_encoding

openc3/lib/openc3/models/gem_model.rb

@@ -39,11 +39,17 @@ class GemModel
     include Api
 
     def self.names
-      result = Pathname.new("#{ENV['GEM_HOME']}/gems").children.select { |c| c.directory? }.collect { |p| File.basename(p) + '.gem' }
+      if Dir.exist?("#{ENV['GEM_HOME']}/gems")
+        result = Pathname.new("#{ENV['GEM_HOME']}/gems").children.select { |c| c.directory? }.collect { |p| File.basename(p) + '.gem' }
+      else
+        result = []
+      end
       return result.sort
     end
 
     def self.get(name)
+      path = "#{ENV['GEM_HOME']}/cosmoscache/#{name}"
+      return path if File.exist?(path)
       path = "#{ENV['GEM_HOME']}/cache/#{name}"
       return path if File.exist?(path)
       raise "Gem #{name} not found"
@@ -52,8 +58,9 @@ def self.get(name)
     def self.put(gem_file_path, gem_install: true, scope:)
       if File.file?(gem_file_path)
         gem_filename = File.basename(gem_file_path)
-        FileUtils.mkdir_p("#{ENV['GEM_HOME']}/cache") unless Dir.exist?("#{ENV['GEM_HOME']}/cache")
-        FileUtils.cp(gem_file_path, "#{ENV['GEM_HOME']}/cache/#{File.basename(gem_file_path)}")
+        # Put into cosmoscache folder that we control
+        FileUtils.mkdir_p("#{ENV['GEM_HOME']}/cosmoscache") unless Dir.exist?("#{ENV['GEM_HOME']}/cosmoscache")
+        FileUtils.cp(gem_file_path, "#{ENV['GEM_HOME']}/cosmoscache/#{File.basename(gem_file_path)}")
         if gem_install
           Logger.info "Installing gem: #{gem_filename}"
           result = OpenC3::ProcessManager.instance.spawn(["ruby", "/openc3/bin/openc3cli", "geminstall", gem_filename, scope], "package_install", gem_filename, Time.now + 3600.0, scope: scope)
@@ -118,5 +125,15 @@ def self.extract_name_and_version(name)
       version = File.basename(split_name[-1], '.gem')
       return gem_name, version
     end
+
+    def self.destroy_all_other_versions(name)
+      keep_gem_name, keep_gem_version = GemModel.extract_name_and_version(name)
+      GemModel.names.each do |gem_full_name|
+        gem_name, gem_version = GemModel.extract_name_and_version(gem_full_name)
+        if gem_name == keep_gem_name and gem_version != keep_gem_version
+          GemModel.destroy(gem_full_name)
+        end
+      end
+    end
   end
 end

openc3/lib/openc3/models/plugin_model.rb

@@ -158,6 +158,11 @@ def self.install_phase2(plugin_hash, scope:, gem_file_path: nil, validate_only:
           gem_file_path = OpenC3::GemModel.get(gem_name)
         end
 
+        # Attempt to remove all older versions of this same plugin before install to prevent version conflicts
+        # Especially on downgrades
+        # Leave the same version if it already exists
+        OpenC3::GemModel.destroy_all_other_versions(File.basename(gem_file_path))
+
         # Actually install the gem now (slow)
         OpenC3::GemModel.install(gem_file_path, scope: scope) unless validate_only
 
@@ -356,7 +361,6 @@ def undeploy
     # Reinstall
     def restore
       plugin_hash = self.as_json(:allow_nan => true)
-      plugin_hash['name'] = plugin_hash['name'].split("__")[0]
       OpenC3::PluginModel.install_phase2(plugin_hash, scope: @scope)
       @destroyed = false
     end

openc3/lib/openc3/models/target_model.rb

@@ -742,7 +742,7 @@ def build_target_archive(temp_dir, target_folder)
       end
     end
 
-    def update_store(system)
+    def update_target_model(system)
       target = system.targets[@name]
 
       # Add in the information from the target and update
@@ -754,10 +754,11 @@ def update_store(system)
       @tlm_unique_id_mode = target.tlm_unique_id_mode
       @limits_groups = system.limits.groups.keys
       update()
+    end
 
-      # Store Packet Definitions
-      system.telemetry.all.each do |target_name, packets|
-        Store.del("#{@scope}__openc3tlm__#{target_name}")
+    def update_store_telemetry(packet_hash, clear_old: true)
+      packet_hash.each do |target_name, packets|
+        Store.del("#{@scope}__openc3tlm__#{target_name}") if clear_old
         packets.each do |packet_name, packet|
           Logger.info "Configuring tlm packet: #{target_name} #{packet_name}"
           begin
@@ -773,8 +774,11 @@ def update_store(system)
           CvtModel.set(json_hash, target_name: packet.target_name, packet_name: packet.packet_name, scope: @scope)
         end
       end
-      system.commands.all.each do |target_name, packets|
-        Store.del("#{@scope}__openc3cmd__#{target_name}")
+    end
+
+    def update_store_commands(packet_hash, clear_old: true)
+      packet_hash.each do |target_name, packets|
+        Store.del("#{@scope}__openc3cmd__#{target_name}") if clear_old
         packets.each do |packet_name, packet|
           Logger.info "Configuring cmd packet: #{target_name} #{packet_name}"
           begin
@@ -785,7 +789,9 @@ def update_store(system)
           end
         end
       end
-      # Store Limits Groups
+    end
+
+    def update_store_limits_groups(system)
       system.limits.groups.each do |group, items|
         begin
           Store.hset("#{@scope}__limits_groups", group, JSON.generate(items))
@@ -794,23 +800,77 @@ def update_store(system)
           raise err
         end
       end
-      # Merge in Limits Sets
+    end
+
+    def update_store_limits_sets(system)
       sets = Store.hgetall("#{@scope}__limits_sets")
       sets ||= {}
       system.limits.sets.each do |set|
         sets[set.to_s] = "false" unless sets.key?(set.to_s)
       end
       Store.hmset("#{@scope}__limits_sets", *sets)
+    end
 
+    def update_store_item_map
       # Create item_map
       item_map_key = "#{@scope}__#{@name}__item_to_packet_map"
       item_map = self.class.build_item_to_packet_map(@name, scope: @scope)
       Store.set(item_map_key, JSON.generate(item_map, :allow_nan => true))
       @@item_map_cache[@name] = [Time.now, item_map]
+    end
 
+    def update_store(system, clear_old: true)
+      update_target_model(system)
+      update_store_telemetry(system.telemetry.all, clear_old: clear_old)
+      update_store_commands(system.commands.all, clear_old: clear_old)
+      update_store_limits_groups(system)
+      update_store_limits_sets(system)
+      update_store_item_map()
       return system
     end
 
+    def dynamic_update(packets, cmd_or_tlm = :TELEMETRY, filename = "dynamic_tlm.txt")
+      # Build hash of targets/packets
+      packet_hash = {}
+      packets.each do |packet|
+        target_name = packet.target_name.upcase
+        packet_hash[target_name] ||= {}
+        packet_name = packet.packet_name.upcase
+        packet_hash[target_name][packet_name] = packet
+      end
+
+      # Update Redis
+      if cmd_or_tlm == :TELEMETRY
+        update_store_telemetry(packet_hash, clear_old: false)
+        update_store_item_map()
+      else
+        update_store_commands(packet_hash, clear_old: false)
+      end
+
+      # Build dynamic file for cmd_tlm
+      configs = {}
+      packets.each do |packet|
+        target_name = packet.target_name.upcase
+        configs[target_name] ||= ""
+        config = configs[target_name]
+        config << packet.to_config(cmd_or_tlm)
+        config << "\n"
+      end
+      configs.each do |target_name, config|
+        begin
+          bucket_key = "#{@scope}/targets_modified/#{target_name}/cmd_tlm/#{filename}"
+          client = Bucket.getClient()
+          client.put_object(
+            # Use targets_modified to save modifications
+            # This keeps the original target clean (read-only)
+            bucket: ENV['OPENC3_CONFIG_BUCKET'],
+            key: bucket_key,
+            body: config
+          )
+        end
+      end
+    end
+
     def deploy_commmandlog_microservice(gem_path, variables, topics, instance = nil, parent = nil)
       microservice_name = "#{@scope}__COMMANDLOG#{instance}__#{@name}"
       microservice = MicroserviceModel.new(
@@ -990,6 +1050,7 @@ def deploy_multi_microservice(gem_path, variables, instance = nil)
           cmd: ["ruby", "multi_microservice.rb", *@children],
           work_dir: '/openc3/lib/openc3/microservices',
           plugin: @plugin,
+          needs_dependencies: @needs_dependencies,
           scope: @scope
         )
         microservice.create