Skip to content

Commit

Permalink
Improving value of remote subsystem details included in ECL and confi…
Browse files Browse the repository at this point in the history 
…g dump entries; Improved TRACE message severities.
  • Loading branch information
@rrivera747
rrivera747 committed Jan 6, 2025
1 parent a0ac4bd commit 1138c48
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 41 deletions.
165 changes: 140 additions & 25 deletions otsdaq/GatewaySupervisor/GatewaySupervisor.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,11 +270,6 @@ void GatewaySupervisor::AppStatusWorkLoop(GatewaySupervisor* theSupervisor)
{
// send back status and progress parameters
const std::string& err = theSupervisor->theStateMachine_.getErrorMessage();

// status = err == "" ? (theSupervisor->theStateMachine_.isInTransition() ? theSupervisor->theStateMachine_.getProvenanceStateName()
// : theSupervisor->theStateMachine_.getCurrentStateName())
// : (theSupervisor->theStateMachine_.getCurrentStateName() == "Paused" ? "Soft-Error:::" : "Failed:::") + err;

try
{
__COUTVS__(39,theSupervisor->theStateMachine_.isInTransition());
Expand Down Expand Up @@ -309,7 +304,8 @@ void GatewaySupervisor::AppStatusWorkLoop(GatewaySupervisor* theSupervisor)
}
else
{
status = (theSupervisor->theStateMachine_.getCurrentStateName() == "Paused" ? "Soft-Error:::" : "Failed:::") + err;
status = (theSupervisor->theStateMachine_.getCurrentStateName() == RunControlStateMachine::PAUSED_STATE_NAME ?
"Soft-Error:::" : "Failed:::") + err;
progress = theSupervisor->theProgressBar_.readPercentageString();
}

Expand All @@ -332,6 +328,22 @@ void GatewaySupervisor::AppStatusWorkLoop(GatewaySupervisor* theSupervisor)
std::lock_guard<std::mutex> lock(theSupervisor->broadcastCommandStatusUpdateMutex_);
if(detail != "" && theSupervisor->broadcastCommandStatus_ != "")
detail += " - " + theSupervisor->broadcastCommandStatus_;

if(!theSupervisor->theStateMachine_.isInTransition() && (
theSupervisor->theStateMachine_.getCurrentStateName() == RunControlStateMachine::CONFIGURED_STATE_NAME ||
theSupervisor->theStateMachine_.getCurrentStateName() == RunControlStateMachine::RUNNING_STATE_NAME ||
theSupervisor->theStateMachine_.getCurrentStateName() == RunControlStateMachine::PAUSED_STATE_NAME))
{
//add Configuration details
detail += " - Configured with System Configuration Alias '" + theSupervisor->activeStateMachineConfigurationAlias_ +
"' which translates to " + theSupervisor->theConfigurationTableGroup_.first + "(" + theSupervisor->theConfigurationTableGroup_.second.str()
+ "). Active Context Group " +
theSupervisor->CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupName(
ConfigurationManager::GroupType::CONTEXT_TYPE) + "(" +
theSupervisor->CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupKey(
ConfigurationManager::GroupType::CONTEXT_TYPE).str() +
").";
}
}
catch(...)
{
Expand Down Expand Up @@ -1351,7 +1363,7 @@ try

//get system messages
value = StringMacros::extractXmlField(remoteStatusString, "systemMessages", 0, after, &after);
__COUTT__ << "Remote System Messages:" << value << __E__;
__COUT_TYPE__(TLVL_DEBUG+2) << __COUT_HDR__ << "Remote System Messages:" << value << __E__;
std::vector<std::string> parsedSysMsgs;
StringMacros::getVectorFromString(value,parsedSysMsgs,{'|'});

Expand All @@ -1366,7 +1378,7 @@ try

//get user with lock
value = StringMacros::extractXmlField(remoteStatusString, "usernameWithLock", 0, after, &after);
__COUTT__ << "Remote User with Lock:" << value << __E__;
__COUT_TYPE__(TLVL_DEBUG+2) << __COUT_HDR__ << "Remote User with Lock:" << value << __E__;
remoteGatewayApp.usernameWithLock = value;

//get Console err/warn count
Expand Down Expand Up @@ -3280,7 +3292,6 @@ try
for(auto& remoteGatewayApp : remoteGatewayApps)
{
if(!remoteGatewayApp.fsm_included) continue; //skip if not included

remoteSubsystemDump += remoteGatewayApp.config_dump;
}

Expand Down Expand Up @@ -3393,6 +3404,40 @@ try
"\n-----------------\n";
else
ss << " No user log entry.";

//insert system and remote subsystem status/detail
{
ss << "\n\n~~~ System Status and Detail ~~~\n";
for(const auto& it : allSupervisorInfo_.getAllSupervisorInfo())
{
const auto& appInfo = it.second;
if(appInfo.getClass() != XDAQContextTable::GATEWAY_SUPERVISOR_CLASS)
continue; //only give Gateway status
ss << "\tStatus: " << appInfo.getStatus() << __E__
<< "\tDetail: " << appInfo.getDetail() << __E__;
}

//also return remote gateways as apps
std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteApps; //local copy
{ //lock for remainder of scope
std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
remoteApps = remoteGatewayApps_;
}

if(remoteApps.size())
{
ss << "\n\n~~~ Subsystem Status and Detail ~~~\n";

for(const auto& remoteApp : remoteApps)
{
const auto& appInfo = remoteApp.appInfo;
ss << "Subsystem Name: " << appInfo.name << __E__
<< "\tStatus: " << appInfo.status << __E__
<< "\tDetail: " << appInfo.detail << __E__;
}
}
}

makeSystemLogEntry(ss.str());
}
__COUT__ << "Done configuring." << __E__;
Expand Down Expand Up @@ -3713,7 +3758,11 @@ try
int dur_h = dur_m / 60;
dur_m = dur_m % 60;
dur_ss << activeStateMachineRunAlias_ << " '" << activeStateMachineRunNumber_ << "' duration so far of " << std::setw(2) << std::setfill('0') << dur_h << ":" << std::setw(2) << std::setfill('0') << dur_m << ":"
<< std::setw(2) << std::setfill('0') << dur_s << "." << dur << " seconds.";
<< std::setw(2) << std::setfill('0') << dur_s; //too much detail "." << dur << " seconds.";
if(dur_h == 0 && dur_m == 0 && dur_s < 5) //if very short, add the detail
dur_ss << "." << dur << " seconds.";
else
dur_ss << ".";

if(doLog)
makeSystemLogEntry("Run pausing. " + dur_ss.str());
Expand Down Expand Up @@ -3953,8 +4002,34 @@ try
if(remoteGatewayApps_.size())
__SUP_COUT_TYPE__(TLVL_DEBUG+22) << __COUT_HDR__ << remoteGatewayApps_[0].command << " " << (remoteGatewayApps_[0].appInfo.status) << __E__;
}

remoteSubsystemDump += "--------------- Remote Subsystem Status ---------------\n";
remoteSubsystemDump += "Remote Subsystem Count: " + std::to_string(remoteGatewayApps.size()) + "\n";
size_t ssi = 1;
for(auto& remoteGatewayApp : remoteGatewayApps)
{
remoteSubsystemDump += std::to_string(ssi) + ". ~~ subsystem_name: " + remoteGatewayApp.appInfo.name + "\n. ";
remoteSubsystemDump += "subsystem_url: " + remoteGatewayApp.appInfo.url + "\n. ";
// remoteSubsystemDump += "subsystem_landingPage: " + remoteGatewayApp.landingPage + "\n. ";
remoteSubsystemDump += "subsystem_status: " + remoteGatewayApp.appInfo.status + "\n. ";
remoteSubsystemDump += "subsystem_progress: " + std::to_string(remoteGatewayApp.appInfo.progress) + "\n. ";
remoteSubsystemDump += "subsystem_detail: " + remoteGatewayApp.appInfo.detail + "\n. ";
// remoteSubsystemDump += "subsystem_lastStatusTime: " + StringMacros::getTimestampString(remoteGatewayApp.appInfo.lastStatusTime) + "\n. ";
// remoteSubsystemDump += "subsystem_consoleErrCount: " + std::to_string(remoteGatewayApp.consoleErrCount) + "\n. ";
// remoteSubsystemDump += "subsystem_consoleWarnCount: " + std::to_string(remoteGatewayApp.consoleWarnCount) + "\n. ";
remoteSubsystemDump += "subsystem_configAlias: " + remoteGatewayApp.selected_config_alias + "\n. ";
remoteSubsystemDump += "subsystem_fsmMode: " + remoteGatewayApp.getFsmMode() + "\n. ";
remoteSubsystemDump += "subsystem_fsmIncluded: " + std::string(remoteGatewayApp.fsm_included?"1":"0") + "\n. ";
}
remoteSubsystemDump += "--------------- end Remote Subsystem Status ---------------\n";

remoteSubsystemDump += "\n\n-----------------\nRemote Configuration dump:\n";
for(auto& remoteGatewayApp : remoteGatewayApps)
{
if(!remoteGatewayApp.fsm_included) continue; //skip if not included
remoteSubsystemDump += remoteGatewayApp.config_dump;
}
remoteSubsystemDump += "\nEND Remote Configuration dump:\n-----------------\n";

if(remoteSubsystemDump.size())
__COUTV__(remoteSubsystemDump);
Expand Down Expand Up @@ -4016,23 +4091,59 @@ try
else
ss << " No user log entry.";

//insert system and remote subsystem status/detail
{
ss << "\n\n~~~ System Status and Detail ~~~\n";
for(const auto& it : allSupervisorInfo_.getAllSupervisorInfo())
{
const auto& appInfo = it.second;
if(appInfo.getClass() != XDAQContextTable::GATEWAY_SUPERVISOR_CLASS)
continue; //only give Gateway status
ss << "\tStatus: " << appInfo.getStatus() << __E__
<< "\tDetail: " << appInfo.getDetail() << __E__;
}

//also return remote gateways as apps
std::vector<GatewaySupervisor::RemoteGatewayInfo> remoteApps; //local copy
{ //lock for remainder of scope
std::lock_guard<std::mutex> lock(remoteGatewayAppsMutex_);
remoteApps = remoteGatewayApps_;
}

ss << "\n\nConfigured with System Configuration Alias '" << activeStateMachineConfigurationAlias_ <<
"' which translates to " << theConfigurationTableGroup_.first << "(" << theConfigurationTableGroup_.second
<< "). Active Context Group " <<
CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupName(
ConfigurationManager::GroupType::CONTEXT_TYPE) << "(" <<
CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupKey(
ConfigurationManager::GroupType::CONTEXT_TYPE) <<
").";

if(activeStateMachineConfigurationDumpOnRunEnable_)
if(remoteApps.size())
{
ss << "\n\n~~~ Subsystem Status and Detail ~~~\n";

for(const auto& remoteApp : remoteApps)
{
const auto& appInfo = remoteApp.appInfo;
ss << "Subsystem Name: " << appInfo.name << __E__
<< "\tStatus: " << appInfo.status << __E__
<< "\tDetail: " << appInfo.detail << __E__;
}
}
}

if(0) //full configuration dump too verbose for ECL (?)
{
ss << "\n\n-----------------\nConfiguration dump:\n" <<
activeStateMachineConfigurationDumpOnRun_;
ss << "\n\nConfigured with System Configuration Alias '" << activeStateMachineConfigurationAlias_ <<
"' which translates to " << theConfigurationTableGroup_.first << "(" << theConfigurationTableGroup_.second
<< "). Active Context Group " <<
CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupName(
ConfigurationManager::GroupType::CONTEXT_TYPE) << "(" <<
CorePropertySupervisorBase::theConfigurationManager_->getActiveGroupKey(
ConfigurationManager::GroupType::CONTEXT_TYPE) <<
").";

if(activeStateMachineConfigurationDumpOnRunEnable_)
{
ss << "\n\n-----------------\nConfiguration dump:\n" <<
activeStateMachineConfigurationDumpOnRun_;
ss << "\nEND Remote Configuration dump:\n-----------------\n";
}

if(remoteSubsystemDump.size())
ss << remoteSubsystemDump;
ss << "\nEND Configuration dump:\n-----------------\n";
}

makeSystemLogEntry(ss.str(),
Expand Down Expand Up @@ -4104,8 +4215,12 @@ try
dur_s = dur_s % 60;
int dur_h = dur_m / 60;
dur_m = dur_m % 60;
dur_ss << activeStateMachineRunAlias_ << " '" << activeStateMachineRunNumber_ << "' duration so far of " << std::setw(2) << std::setfill('0') << dur_h << ":" << std::setw(2) << std::setfill('0') << dur_m << ":"
<< std::setw(2) << std::setfill('0') << dur_s << "." << dur << " seconds.";
dur_ss << activeStateMachineRunAlias_ << " '" << activeStateMachineRunNumber_ << "' duration of " << std::setw(2) << std::setfill('0') << dur_h << ":" << std::setw(2) << std::setfill('0') << dur_m << ":"
<< std::setw(2) << std::setfill('0') << dur_s; //too much detail "." << dur << " seconds.";
if(dur_h == 0 && dur_m == 0 && dur_s < 5) //if very short, add the detail
dur_ss << "." << dur << " seconds.";
else
dur_ss << ".";

if(doLog)
{
Expand Down
37 changes: 22 additions & 15 deletions otsdaq/Macros/StringMacros.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,16 +189,13 @@ void StringMacros::sanitizeForSQL(std::string& str)
// if(allowWhiteSpace) convert \t to 8 &#160; spaces and \n to <br>
std::string StringMacros::escapeString(std::string inString, bool allowWhiteSpace /* = false */)
{
bool doit = false;

unsigned int ws = -1;
char htmlTmp[10];

for(unsigned int i = 0; i < inString.length(); i++)
if(inString[i] != ' ')
{
if(doit)
__COUT__ << inString[i] << ":" << (int)inString[i] << ":" << inString << std::endl;
__COUT_TYPE__(TLVL_DEBUG+30) << __COUT_HDR__ << i << ". " << inString[i] << ":" << (int)inString[i] << std::endl;

// remove new lines and unprintable characters
if(inString[i] == '\r' || inString[i] == '\n' || // remove new line chars
Expand All @@ -211,6 +208,21 @@ std::string StringMacros::escapeString(std::string inString, bool allowWhiteSpac
// impossible if by byte (but there are html
// chracters in 300s and 8000s)
{
//handle UTF-8 encoded characters
if(i+2 < inString.size() && inString[i] == char(0xE2) &&
inString[i+1] == char(0x80) && inString[i+2] == char(0x93)) // longer dash endash is 3-bytes 0xE2 0x80 0x93
{
//encode "--" as &#8211;
inString.insert(i, "&#82"); // insert HTML name before special character
inString.replace(i + 4, 1, 1, '1'); // replace special character-0 with s
inString.replace(i + 5, 1, 1, '1'); // replace special character-1 with h
inString.replace(i + 6, 1, 1, ';'); // replace special character-2 with ;
i += 7; // skip to next char to check
ws = i; // last non white space char
--i;
continue;
}

if( // maintain new lines and tabs
inString[i] == '\n')
{
Expand Down Expand Up @@ -256,13 +268,11 @@ std::string StringMacros::escapeString(std::string inString, bool allowWhiteSpac
inString.erase(i, 1); // erase character
--i; // step back so next char to check is correct
}
if(doit)
__COUT__ << inString << std::endl;
__COUT_TYPE__(TLVL_DEBUG+31) << __COUT_HDR__ << inString << std::endl;
continue;
}

if(doit)
__COUT__ << inString << std::endl;
__COUT_TYPE__(TLVL_DEBUG+31) << __COUT_HDR__ << inString << std::endl;

// replace special characters
if(inString[i] == '\"' || inString[i] == '\'')
Expand All @@ -272,7 +282,7 @@ std::string StringMacros::escapeString(std::string inString, bool allowWhiteSpac
inString.replace(i + 5, 1, 1, ';'); // replace special character with ;
i += 5; // skip to next char to check
//__COUT__ << inString << std::endl;
}
}
else if(inString[i] == '&')
{
inString.insert(i, "&amp"); // insert HTML name before special character
Expand All @@ -294,8 +304,7 @@ std::string StringMacros::escapeString(std::string inString, bool allowWhiteSpac
i += 5; // skip to next char to check
}

if(doit)
__COUT__ << inString << std::endl;
__COUT_TYPE__(TLVL_DEBUG+30) << __COUT_HDR__ << inString << std::endl;

ws = i; // last non white space char
}
Expand All @@ -316,13 +325,11 @@ std::string StringMacros::escapeString(std::string inString, bool allowWhiteSpac
// ws = i;
}

if(doit)
__COUT__ << inString.size() << " " << ws << std::endl;
__COUT_TYPE__(TLVL_DEBUG+30) << __COUT_HDR__ << inString.size() << " " << ws << std::endl;

// inString.substr(0,ws+1);

if(doit)
__COUT__ << inString.size() << " " << inString << std::endl;
__COUT_TYPE__(TLVL_DEBUG+30) << __COUT_HDR__ << inString.size() << " " << inString << std::endl;

if(allowWhiteSpace) // keep all white space
return inString;
Expand Down
3 changes: 2 additions & 1 deletion otsdaq/TableCore/TableGroupKey.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ class TableGroupKey

unsigned int key(void) const;
bool isInvalid(void) const;
std::string toString(void) const;
std::string toString(void) const;
std::string str() const { return toString(); } // alternative alias method

// Operators
TableGroupKey& operator=(const unsigned int key);
Expand Down

0 comments on commit 1138c48

Please sign in to comment.