Merge pull request #130 from DUNE-DAQ/eflumerf/IOManager_OKS

Introduce IOManager configuration object
DUNE-DAQ · Oct 14, 2024 · 4ecca2e · 4ecca2e
2 parents 042d032 + 05a208b
commit 4ecca2e
Show file tree

Hide file tree

Showing 10 changed files with 65 additions and 50 deletions.
diff --git a/config/daqsystemtest/ccm.data.xml b/config/daqsystemtest/ccm.data.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="ASCII"?>
+<?xml version="1.0" encoding="us-ascii"?>
 
 <!-- oks-data version 2.2 -->
 
@@ -63,7 +63,7 @@
 
 <oks-data>
 
-<info name="" type="" num-of-items="25" oks-format="data" oks-version="862f2957270" created-by="gjc" created-on="thinkpad" creation-time="20231207T110859" last-modified-by="glehmann" last-modified-on="np04-srv-024.cern.ch" last-modification-time="20240923T065642"/>
+<info name="" type="" num-of-items="28" oks-format="data" oks-version="862f2957270" created-by="gjc" created-on="thinkpad" creation-time="20231207T110859" last-modified-by="eflumerf" last-modified-on="ironvirt9.mshome.net" last-modification-time="20241011T131354"/>
 
 <include>
  <file path="schema/confmodel/dunedaq.schema.xml"/>
@@ -83,7 +83,7 @@
  <attr name="commandline_parameters" type="string">
   <data val="Process PID $$;"/>
   <data val="trap &apos;pkill -INT -P $$&apos; EXIT;"/>
-  <data val="gunicorn -b 0.0.0.0:${CONNECTION_PORT} --workers=1 --worker-class=gthread --threads=2 --timeout 5000000000 --log-level=debug connection-service.connection-flask:app"/>
+  <data val="gunicorn -b 0.0.0.0:5000 --workers=1 --worker-class=gthread --threads=2 --timeout 5000000000 --log-level=debug connection-service.connection-flask:app"/>
  </attr>
  <attr name="threads" type="u16" val="1"/>
  <rel name="runs_on" class="VirtualHost" id="vlocalhost"/>
@@ -93,6 +93,18 @@
  <rel name="opmon_conf" class="OpMonConf" id="slow-all-monitoring"/>
 </obj>
 
+<obj class="ConnectivityService" id="ehn1-connectivity-service-config">
+ <attr name="interval_ms" type="u32" val="2000"/>
+ <attr name="host" type="string" val="np04-srv-017"/>
+ <rel name="service" class="Service" id="ehn1-connectivity-service"/>
+</obj>
+
+<obj class="ConnectivityService" id="local-connectivity-service-config">
+ <attr name="interval_ms" type="u32" val="2000"/>
+ <attr name="host" type="string" val="localhost"/>
+ <rel name="service" class="Service" id="local-connectivity-service"/>
+</obj>
+
 <obj class="OpMonConf" id="fast-all-monitoring">
  <attr name="level" type="u32" val="4294967295"/>
  <attr name="interval_s" type="u32" val="2"/>
@@ -123,6 +135,11 @@
  <rel name="fsm" class="FSMconfiguration" id="fsmConf-test"/>
 </obj>
 
+<obj class="Service" id="ehn1-connectivity-service">
+ <attr name="protocol" type="string" val="http"/>
+ <attr name="port" type="u16" val="30005"/>
+</obj>
+
 <obj class="Service" id="local-connectivity-service">
  <attr name="protocol" type="string" val="http"/>
  <attr name="port" type="u16" val="5000"/>
@@ -174,16 +191,6 @@
  <attr name="value" type="string" val="erstrace,throttle,lstdout,protobufstream(monkafka.cern.ch:30092)"/>
 </obj>
 
-<obj class="Variable" id="local-env-connectivity-port">
- <attr name="name" type="string" val="CONNECTION_PORT"/>
- <attr name="value" type="string" val="5000"/>
-</obj>
-
-<obj class="Variable" id="local-env-connectivity-server">
- <attr name="name" type="string" val="CONNECTION_SERVER"/>
- <attr name="value" type="string" val="localhost"/>
-</obj>
-
 <obj class="Variable" id="local-env-ers-error">
  <attr name="name" type="string" val="DUNEDAQ_ERS_ERROR"/>
  <attr name="value" type="string" val="erstrace,throttle,lstdout"/>
@@ -211,8 +218,6 @@
 
 <obj class="VariableSet" id="ehn1-variables">
  <rel name="contains">
-  <ref class="Variable" id="ehn1-env-connectivity-port"/>
-  <ref class="Variable" id="ehn1-env-connectivity-server"/>
   <ref class="Variable" id="ehn1-env-ers-error"/>
   <ref class="Variable" id="ehn1-env-ers-fatal"/>
   <ref class="Variable" id="ehn1-env-ers-info"/>
@@ -224,8 +229,6 @@
 
 <obj class="VariableSet" id="local-variables">
  <rel name="contains">
-  <ref class="Variable" id="local-env-connectivity-port"/>
-  <ref class="Variable" id="local-env-connectivity-server"/>
   <ref class="Variable" id="local-env-ers-error"/>
   <ref class="Variable" id="local-env-ers-fatal"/>
   <ref class="Variable" id="local-env-ers-info"/>

diff --git a/config/daqsystemtest/example-configs.data.xml b/config/daqsystemtest/example-configs.data.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="us-ascii"?>
+<?xml version="1.0" encoding="ASCII"?>
 
 <!-- oks-data version 2.2 -->
 
@@ -63,7 +63,7 @@
 
 <oks-data>
 
-<info name="" type="" num-of-items="15" oks-format="data" oks-version="862f2957270" created-by="gjc" created-on="thinkpad" creation-time="20231207T105549" last-modified-by="maroda" last-modified-on="np04-srv-015.cern.ch" last-modification-time="20240918T124332"/>
+<info name="" type="" num-of-items="6" oks-format="data" oks-version="862f2957270" created-by="gjc" created-on="thinkpad" creation-time="20231207T105549" last-modified-by="eflumerf" last-modified-on="ironvirt9.IRONDOMAIN.local" last-modification-time="20241014T145531"/>
 
 <include>
  <file path="schema/confmodel/dunedaq.schema.xml"/>
@@ -87,6 +87,7 @@
  <attr name="tpg_channel_map" type="string" val="PD2HDChannelMap"/>
  <attr name="clock_speed_hz" type="u32" val="62500000"/>
  <attr name="op_env" type="string" val="test"/>
+ <attr name="offline_data_stream" type="string" val="cosmics"/>
 </obj>
 
 <obj class="Segment" id="root-segment">
@@ -100,10 +101,10 @@
 </obj>
 
 <obj class="Session" id="ehn1-local-1x1-config">
- <attr name="use_connectivity_server" type="bool" val="1"/>
- <attr name="connectivity_service_interval_ms" type="u32" val="2000"/>
  <attr name="data_request_timeout_ms" type="u32" val="1000"/>
  <attr name="data_rate_slowdown_factor" type="u32" val="1"/>
+ <attr name="controller_log_level" type="enum" val="INFO"/>
+ <rel name="connectivity_service" class="ConnectivityService" id="ehn1-connectivity-service-config"/>
  <rel name="environment">
   <ref class="VariableSet" id="ehn1-variables"/>
  </rel>
@@ -118,10 +119,10 @@
 </obj>
 
 <obj class="Session" id="ehn1-local-2x3-config">
- <attr name="use_connectivity_server" type="bool" val="1"/>
- <attr name="connectivity_service_interval_ms" type="u32" val="2000"/>
  <attr name="data_request_timeout_ms" type="u32" val="1000"/>
  <attr name="data_rate_slowdown_factor" type="u32" val="1"/>
+ <attr name="controller_log_level" type="enum" val="INFO"/>
+ <rel name="connectivity_service" class="ConnectivityService" id="ehn1-connectivity-service-config"/>
  <rel name="environment">
   <ref class="VariableSet" id="ehn1-variables"/>
  </rel>
@@ -131,18 +132,16 @@
 </obj>
 
 <obj class="Session" id="local-1x1-config">
- <attr name="use_connectivity_server" type="bool" val="1"/>
- <attr name="connectivity_service_interval_ms" type="u32" val="2000"/>
  <attr name="data_request_timeout_ms" type="u32" val="1000"/>
  <attr name="data_rate_slowdown_factor" type="u32" val="1"/>
+ <attr name="controller_log_level" type="enum" val="INFO"/>
+ <rel name="connectivity_service" class="ConnectivityService" id="local-connectivity-service-config"/>
  <rel name="environment">
   <ref class="Variable" id="local-env-ers-verb"/>
   <ref class="Variable" id="local-env-ers-info"/>
   <ref class="Variable" id="local-env-ers-warning"/>
   <ref class="Variable" id="local-env-ers-error"/>
   <ref class="Variable" id="local-env-ers-fatal"/>
-  <ref class="Variable" id="local-env-connectivity-server"/>
-  <ref class="Variable" id="local-env-connectivity-port"/>
  </rel>
  <rel name="disabled">
   <ref class="DFApplication" id="df-02"/>
@@ -158,18 +157,16 @@
 </obj>
 
 <obj class="Session" id="local-2x3-config">
- <attr name="use_connectivity_server" type="bool" val="1"/>
- <attr name="connectivity_service_interval_ms" type="u32" val="2000"/>
  <attr name="data_request_timeout_ms" type="u32" val="1000"/>
  <attr name="data_rate_slowdown_factor" type="u32" val="1"/>
+ <attr name="controller_log_level" type="enum" val="INFO"/>
+ <rel name="connectivity_service" class="ConnectivityService" id="local-connectivity-service-config"/>
  <rel name="environment">
   <ref class="Variable" id="local-env-ers-verb"/>
   <ref class="Variable" id="local-env-ers-info"/>
   <ref class="Variable" id="local-env-ers-warning"/>
   <ref class="Variable" id="local-env-ers-error"/>
   <ref class="Variable" id="local-env-ers-fatal"/>
-  <ref class="Variable" id="local-env-connectivity-server"/>
-  <ref class="Variable" id="local-env-connectivity-port"/>
  </rel>
  <rel name="segment" class="Segment" id="root-segment"/>
  <rel name="infrastructure_applications">
@@ -179,5 +176,4 @@
  <rel name="opmon_uri" class="OpMonURI" id="local-opmon-uri"/>
 </obj>
 
-
 </oks-data>
diff --git a/integtest/3ru_1df_multirun_test.py b/integtest/3ru_1df_multirun_test.py
@@ -78,10 +78,12 @@
 }
 ignored_logfile_problems = {
     "-controller": [
-        "Propagating take_control to children",
-        "There is no broadcasting service",
+        'ERROR.*Broadcast:.*Propagating take_control to children',
+        'ERROR.*Broadcast:.*Propagating describe to children',
+        'WARNING.*Broadcast:.*There is no broadcasting service!',
         "Could not understand the BroadcastHandler technology you want to use",
         "Worker with pid \\d+ was terminated due to signal 1",
+        "Connection '.*' not found on the application registry",
     ],
     "local-connection-server": [
         "errorlog: -",

diff --git a/integtest/3ru_3df_multirun_test.py b/integtest/3ru_3df_multirun_test.py
@@ -76,10 +76,12 @@
 }
 ignored_logfile_problems = {
     "-controller": [
-        "Propagating take_control to children",
-        "There is no broadcasting service",
+        'ERROR.*Broadcast:.*Propagating take_control to children',
+        'ERROR.*Broadcast:.*Propagating describe to children',
+        'WARNING.*Broadcast:.*There is no broadcasting service!',
         "Could not understand the BroadcastHandler technology you want to use",
         "Worker with pid \\d+ was terminated due to signal 1",
+        "Connection '.*' not found on the application registry",
     ],
     "local-connection-server": [
         "errorlog: -",

diff --git a/integtest/example_system_test.py b/integtest/example_system_test.py
@@ -51,10 +51,12 @@
 }
 ignored_logfile_problems = {
     "-controller": [
-        'ERROR    "Broadcast": Propagating take_control to children',
-        'WARNING  "Broadcast": There is no broadcasting service!',
+        'ERROR.*Broadcast:.*Propagating take_control to children',
+        'ERROR.*Broadcast:.*Propagating describe to children',
+        'WARNING.*Broadcast:.*There is no broadcasting service!',
         "Worker with pid \\d+ was terminated due to signal",
-        'WARNING  "BroadcastHandler": Could not understand the BroadcastHandler technology you want to use',
+        "Could not understand the BroadcastHandler technology you want to use",
+        "Connection '.*' not found on the application registry",
     ],
     "local-connection-server": [
         "errorlog: -",

diff --git a/integtest/long_window_readout_test.py b/integtest/long_window_readout_test.py
@@ -61,10 +61,12 @@
 }
 ignored_logfile_problems = {
     "-controller": [
-        "Propagating take_control to children",
-        "There is no broadcasting service",
+        'ERROR.*Broadcast:.*Propagating take_control to children',
+        'ERROR.*Broadcast:.*Propagating describe to children',
+        'WARNING.*Broadcast:.*There is no broadcasting service!',
         "Could not understand the BroadcastHandler technology you want to use",
         "Worker with pid \\d+ was terminated due to signal 1",
+        "Connection '.*' not found on the application registry",
     ],
     "local-connection-server": [
         "errorlog: -",

diff --git a/integtest/minimal_system_quick_test.py b/integtest/minimal_system_quick_test.py
@@ -61,10 +61,12 @@
 }
 ignored_logfile_problems = {
     "-controller": [
-        'ERROR    "Broadcast": Propagating take_control to children',
-        'WARNING  "Broadcast": There is no broadcasting service!',
+        'ERROR.*Broadcast:.*Propagating take_control to children',
+        'ERROR.*Broadcast:.*Propagating describe to children',
+        'WARNING.*Broadcast:.*There is no broadcasting service!',
         "Worker with pid \\d+ was terminated due to signal",
-        'WARNING  "BroadcastHandler": Could not understand the BroadcastHandler technology you want to use',
+        "Could not understand the BroadcastHandler technology you want to use",
+        "Connection '.*' not found on the application registry",
     ],
     "local-connection-server": [
         "errorlog: -",

diff --git a/integtest/readout_type_scan.py b/integtest/readout_type_scan.py
@@ -105,10 +105,12 @@
 }
 ignored_logfile_problems = {
     "-controller": [
-        "Propagating take_control to children",
-        "There is no broadcasting service",
+        'ERROR.*Broadcast:.*Propagating take_control to children',
+        'ERROR.*Broadcast:.*Propagating describe to children',
+        'WARNING.*Broadcast:.*There is no broadcasting service!',
         "Could not understand the BroadcastHandler technology you want to use",
         "Worker with pid \\d+ was terminated due to signal 1",
+        "Connection '.*' not found on the application registry",
     ],
     "local-connection-server": [
         "errorlog: -",

diff --git a/integtest/small_footprint_quick_test.py b/integtest/small_footprint_quick_test.py
@@ -64,10 +64,12 @@
         "Searching for connections matching uid_regex<errored_frames_q> and data_type Unknown"
     ],
     "-controller": [
-        "Propagating take_control to children",
-        "There is no broadcasting service",
+        'ERROR.*Broadcast:.*Propagating take_control to children',
+        'ERROR.*Broadcast:.*Propagating describe to children',
+        'WARNING.*Broadcast:.*There is no broadcasting service!',
         "Could not understand the BroadcastHandler technology you want to use",
         "Worker with pid \\d+ was terminated due to signal 1",
+        "Connection '.*' not found on the application registry",
     ],
     "local-connection-server": [
         "errorlog: -",

diff --git a/integtest/tpstream_writing_test.py b/integtest/tpstream_writing_test.py
@@ -83,10 +83,12 @@
 }
 ignored_logfile_problems = {
     "-controller": [
-        "Propagating take_control to children",
-        "There is no broadcasting service",
+        'ERROR.*Broadcast:.*Propagating take_control to children',
+        'ERROR.*Broadcast:.*Propagating describe to children',
+        'WARNING.*Broadcast:.*There is no broadcasting service!',
         "Could not understand the BroadcastHandler technology you want to use",
         "Worker with pid \\d+ was terminated due to signal 1",
+        "Connection '.*' not found on the application registry",
     ],
     "local-connection-server": [
         "errorlog: -",