From a8aa9e7303a96bec39c319c2a6399d0c15f976b8 Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Fri, 17 Dec 2021 21:40:01 +0100 Subject: [PATCH 01/14] Real time logging now working --- PILOTVERSION | 2 +- pilot/util/constants.py | 2 +- pilot/util/realtimelogger.py | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/PILOTVERSION b/PILOTVERSION index 27d4f6b57..d12cba2da 100644 --- a/PILOTVERSION +++ b/PILOTVERSION @@ -1 +1 @@ -3.1.0.63 \ No newline at end of file +3.1.0.64 \ No newline at end of file diff --git a/pilot/util/constants.py b/pilot/util/constants.py index 34644997b..44f1f841f 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -14,7 +14,7 @@ RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates REVISION = '0' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '63' # build number should be reset to '1' for every new development cycle +BUILD = '64' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 diff --git a/pilot/util/realtimelogger.py b/pilot/util/realtimelogger.py index c6f569bea..4649294b8 100644 --- a/pilot/util/realtimelogger.py +++ b/pilot/util/realtimelogger.py @@ -107,18 +107,16 @@ def __init__(self, args, info_dic, level=INFO): server, port, timeout=5.0, - ssl_enable=False, + ssl_enable=True, ssl_verify=False, - user='pilot', + username='pilot', password='XXX' ) # Create the handler _handler = AsynchronousLogstashHandler( - host=server, - port=port, + server, + port, transport=transport, - ssl_enable=False, - ssl_verify=False, database_path='logstash_test.db') else: logger.warning(f'unknown logtype: {logtype}') @@ -137,7 +135,8 @@ def set_jobinfo(self, job): self.jobinfo = {"TaskID": job.taskid, "PandaJobID": job.jobid} if 'HARVESTER_WORKER_ID' in os.environ: self.jobinfo["Harvester_WorkerID"] = os.environ.get('HARVESTER_WORKER_ID') - logger.debug('set_jobinfo with PandaJobID=%s', self.jobinfo["PandaJobID"]) + if 'HARVESTER_ID' in os.environ: + self.jobinfo["Harvester_ID"] = os.environ.get('HARVESTER_ID') # prepend some panda job info # check if the msg is a dict-based object via isinstance(msg,dict), From 912bbc7db71de1cdba969fa7f4c5aee2770de12f Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Fri, 14 Jan 2022 14:30:45 +0100 Subject: [PATCH 02/14] Added *.part* to redundants list --- pilot/user/atlas/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pilot/user/atlas/common.py b/pilot/user/atlas/common.py index 2e4e6271e..1a71734c6 100644 --- a/pilot/user/atlas/common.py +++ b/pilot/user/atlas/common.py @@ -1887,6 +1887,7 @@ def get_redundants(): "singularity/*", "/cores", "/work", + "*.part*", "docs/", "/pilot3"] From 5189cbd4b18614ab426286c3cb6863e1fe51db79 Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Sun, 16 Jan 2022 16:50:26 +0100 Subject: [PATCH 03/14] Version --- PILOTVERSION | 2 +- pilot/util/constants.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/PILOTVERSION b/PILOTVERSION index d12cba2da..164a848c6 100644 --- a/PILOTVERSION +++ b/PILOTVERSION @@ -1 +1 @@ -3.1.0.64 \ No newline at end of file +3.1.1.1 \ No newline at end of file diff --git a/pilot/util/constants.py b/pilot/util/constants.py index 44f1f841f..f12346788 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -13,8 +13,8 @@ # Pilot version RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates -REVISION = '0' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '64' # build number should be reset to '1' for every new development cycle +REVISION = '1' # revision number should be reset to '0' for every new version release, increased for small updates +BUILD = '1' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 From 2cf631227d14cc14573c8da76564030ef5898be1 Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Wed, 19 Jan 2022 10:14:30 +0100 Subject: [PATCH 04/14] Removed atlasLocalSetup from arcproxy setup --- PILOTVERSION | 2 +- pilot/user/atlas/proxy.py | 2 +- pilot/util/constants.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/PILOTVERSION b/PILOTVERSION index 164a848c6..2fc27ca38 100644 --- a/PILOTVERSION +++ b/PILOTVERSION @@ -1 +1 @@ -3.1.1.1 \ No newline at end of file +3.1.1.2 \ No newline at end of file diff --git a/pilot/user/atlas/proxy.py b/pilot/user/atlas/proxy.py index 2f5496aa6..fbbda32ca 100644 --- a/pilot/user/atlas/proxy.py +++ b/pilot/user/atlas/proxy.py @@ -44,7 +44,7 @@ def verify_proxy(limit=None, x509=None, proxy_id="pilot", test=False): else: envsetup = '' - envsetup += ". %s/atlas.cern.ch/repo/ATLASLocalRootBase/user/atlasLocalSetup.sh --quiet;" % get_file_system_root_path() + # envsetup += ". %s/atlas.cern.ch/repo/ATLASLocalRootBase/user/atlasLocalSetup.sh --quiet;" % get_file_system_root_path() if os.environ.get('ALRB_noGridMW', '').lower() != "yes": envsetup += "lsetup emi;" else: diff --git a/pilot/util/constants.py b/pilot/util/constants.py index f12346788..72969d8e8 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -14,7 +14,7 @@ RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates REVISION = '1' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '1' # build number should be reset to '1' for every new development cycle +BUILD = '2' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 From 29ebd13c50c47c7ae6e1a4a28459ffcb0dfcc3e5 Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Wed, 19 Jan 2022 17:03:16 +0100 Subject: [PATCH 05/14] Removed lsetup before arcproxy call. Protected /proc/%d/stat file open with try-statement --- PILOTVERSION | 2 +- pilot/user/atlas/proxy.py | 8 ++++---- pilot/util/constants.py | 2 +- pilot/util/processes.py | 9 ++++++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/PILOTVERSION b/PILOTVERSION index 2fc27ca38..f927970a8 100644 --- a/PILOTVERSION +++ b/PILOTVERSION @@ -1 +1 @@ -3.1.1.2 \ No newline at end of file +3.1.1.4 \ No newline at end of file diff --git a/pilot/user/atlas/proxy.py b/pilot/user/atlas/proxy.py index fbbda32ca..29373584b 100644 --- a/pilot/user/atlas/proxy.py +++ b/pilot/user/atlas/proxy.py @@ -45,10 +45,10 @@ def verify_proxy(limit=None, x509=None, proxy_id="pilot", test=False): envsetup = '' # envsetup += ". %s/atlas.cern.ch/repo/ATLASLocalRootBase/user/atlasLocalSetup.sh --quiet;" % get_file_system_root_path() - if os.environ.get('ALRB_noGridMW', '').lower() != "yes": - envsetup += "lsetup emi;" - else: - logger.warning('Skipping "lsetup emi" as ALRB_noGridMW=YES') + #if os.environ.get('ALRB_noGridMW', '').lower() != "yes": + # envsetup += "lsetup emi;" + #else: + # logger.warning('Skipping "lsetup emi" as ALRB_noGridMW=YES') # first try to use arcproxy since voms-proxy-info is not working properly on SL6 # (memory issues on queues with limited memory) diff --git a/pilot/util/constants.py b/pilot/util/constants.py index 72969d8e8..c456d8132 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -14,7 +14,7 @@ RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates REVISION = '1' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '2' # build number should be reset to '1' for every new development cycle +BUILD = '4' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 diff --git a/pilot/util/processes.py b/pilot/util/processes.py index a158d3f98..2b0e39970 100644 --- a/pilot/util/processes.py +++ b/pilot/util/processes.py @@ -523,9 +523,12 @@ def get_instant_cpu_consumption_time(pid): if pid and hz and hz > 0: path = "/proc/%d/stat" % pid if os.path.exists(path): - with open(path) as fp: - fields = fp.read().split(' ')[13:17] - utime, stime, cutime, cstime = [(float(f) / hz) for f in fields] + try: + with open(path) as fp: + fields = fp.read().split(' ')[13:17] + utime, stime, cutime, cstime = [(float(f) / hz) for f in fields] + except (FileNotFoundError, IOError) as exc: + logger.warning(f'exception caught: {exc} (ignored)') if utime and stime and cutime and cstime: # sum up all the user+system times for both the main process (pid) and the child processes From 4303709871ec7b27a45c62f4b9aa4f9406419851 Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Thu, 20 Jan 2022 16:41:23 +0100 Subject: [PATCH 06/14] Added iterative function to determine a good fitting range for memory leak fits --- pilot/api/analytics.py | 88 ++++++++++++++++++++++++++++++++++++--- pilot/user/atlas/proxy.py | 2 +- pilot/util/constants.py | 4 +- pilot/util/math.py | 2 +- pilot/util/processes.py | 2 +- 5 files changed, 88 insertions(+), 10 deletions(-) diff --git a/pilot/api/analytics.py b/pilot/api/analytics.py index 8af4f4e12..311f0aa40 100644 --- a/pilot/api/analytics.py +++ b/pilot/api/analytics.py @@ -138,6 +138,13 @@ def get_fitted_data(self, filename, x_name='Time', y_name='pss+swap', precision= # remove tails if desired # this is useful e.g. for memory monitor data where the first and last values # represent allocation and de-allocation, ie not interesting + + itmet = False + if len(x) >= 100: + logger.debug('tails will not be removed for large data sample - iterative method will be used instead') + tails = True + itmet = True + if not tails and len(x) > 7 and len(y) > 7: logger.debug('removing tails from data to be fitted') x = x[5:] @@ -145,24 +152,95 @@ def get_fitted_data(self, filename, x_name='Time', y_name='pss+swap', precision= y = y[5:] y = y[:-2] - if (len(x) > 7 and len(y) > 7) and len(x) == len(y): + if not (len(x) > 7 and len(y) > 7) and len(x) == len(y): + logger.warning('wrong length of table data, x=%s, y=%s (must be same and length>=4)', str(x), str(y)) + else: logger.info('fitting %s vs %s', y_name, x_name) + + if itmet: + norg = len(x) + fit = self.fit(x, y) + _slope = self.slope() + _chi2_org = fit.chi2() + + # determine the removable right region ("right side limit") + _x = x + _y = y + right_limit = self.find_limit(_x, _y, _chi2_org, norg, edge="right") + + # determine the removable left region ("left side limit") + _x = x + _y = y + left_limit = self.find_limit(_x, _y, _chi2_org, norg, edge="left") + + # final fit adjusted for removable regions + x = x[left_limit:right_limit] + y = y[left_limit:right_limit] + try: fit = self.fit(x, y) _slope = self.slope() - except Exception as e: - logger.warning('failed to fit data, x=%s, y=%s: %s', str(x), str(y), e) + except Exception as exc: + logger.warning('failed to fit data, x=%s, y=%s: %s', str(x), str(y), exc) else: if _slope: slope = float_to_rounded_string(fit.slope(), precision=precision) chi2 = float_to_rounded_string(fit.chi2(), precision=precision) if slope != "": logger.info('current memory leak: %s B/s (using %d data points, chi2=%s)', slope, len(x), chi2) - else: - logger.warning('wrong length of table data, x=%s, y=%s (must be same and length>=4)', str(x), str(y)) return {"slope": slope, "chi2": chi2} + def find_limit(self, _x, _y, _chi2_org, norg, change_limit=0.25, edge="right", steps=5): + """ + Use an iterative approach to find the limits of the distributions that can be used for the final fit. + """ + + limit = 0 + _chi2_prev = _chi2_org + found = False + iterations = 0 + while len(_x) > norg / 2: + iterations += 1 + if edge == "right": + _x = _x[:-steps] + _y = _y[:-steps] + else: # left edge + _x = _x[steps:] + _y = _y[steps:] + try: + fit = self.fit(_x, _y) + _slope = self.slope() + except Exception as exc: + logger.warning(f'caught exception: {exc}') + break + + _chi2 = fit.chi2() + change = (_chi2_prev - _chi2) / _chi2_prev + logger.info(f'current chi2={_chi2} (change={change * 100} %)') + if change < change_limit: + found = True + break + else: + _chi2_prev = _chi2 + + if edge == "right": + if not found: + limit = norg - 1 + logger.warning('right removable region not found') + else: + limit = len(_x) - 1 + logger.info(f'right removable region: {limit}') + else: + if not found: + limit = 0 + logger.info('left removable region not found') + else: + limit = iterations * 10 + logger.info(f'left removable region: {limit}') + + return limit + def extract_from_table(self, table, x_name, y_name): """ diff --git a/pilot/user/atlas/proxy.py b/pilot/user/atlas/proxy.py index 29373584b..2f736b853 100644 --- a/pilot/user/atlas/proxy.py +++ b/pilot/user/atlas/proxy.py @@ -5,7 +5,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 # # Authors: -# - Paul Nilsson, paul.nilsson@cern.ch, 2018 +# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2022 # - Alexander Bogdanchikov, Alexander.Bogdanchikov@cern.ch, 2020 # from pilot.util.container import execute diff --git a/pilot/util/constants.py b/pilot/util/constants.py index c456d8132..a329c818c 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -6,7 +6,7 @@ # # Authors: # - Mario Lassnig, mario.lassnig@cern.ch, 2017 -# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2021 +# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2022 from os import environ @@ -14,7 +14,7 @@ RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates REVISION = '1' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '4' # build number should be reset to '1' for every new development cycle +BUILD = '5' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 diff --git a/pilot/util/math.py b/pilot/util/math.py index 33ec2e2d1..3449eecec 100644 --- a/pilot/util/math.py +++ b/pilot/util/math.py @@ -82,7 +82,7 @@ def chi2(observed, expected): if 0 in expected: return 0.0 - return sum((_o - _e) ** 2 / _e for _o, _e in zip(observed, expected)) + return sum((_o - _e) ** 2 / _e ** 2 for _o, _e in zip(observed, expected)) def float_to_rounded_string(num, precision=3): diff --git a/pilot/util/processes.py b/pilot/util/processes.py index 2b0e39970..14342438a 100644 --- a/pilot/util/processes.py +++ b/pilot/util/processes.py @@ -5,7 +5,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 # # Authors: -# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2021 +# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2022 import os import time From ad272c565ae6f0947f997f5a7e460cce281af4a5 Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Fri, 21 Jan 2022 09:50:27 +0100 Subject: [PATCH 07/14] Stopping the edge iteration when the data is reduced to 2/3 of the original size --- PILOTVERSION | 2 +- pilot/api/analytics.py | 2 +- pilot/util/constants.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/PILOTVERSION b/PILOTVERSION index f927970a8..861a59a15 100644 --- a/PILOTVERSION +++ b/PILOTVERSION @@ -1 +1 @@ -3.1.1.4 \ No newline at end of file +3.1.1.6 \ No newline at end of file diff --git a/pilot/api/analytics.py b/pilot/api/analytics.py index 311f0aa40..b4cd350df 100644 --- a/pilot/api/analytics.py +++ b/pilot/api/analytics.py @@ -200,7 +200,7 @@ def find_limit(self, _x, _y, _chi2_org, norg, change_limit=0.25, edge="right", s _chi2_prev = _chi2_org found = False iterations = 0 - while len(_x) > norg / 2: + while len(_x) > 2 * norg / 3: iterations += 1 if edge == "right": _x = _x[:-steps] diff --git a/pilot/util/constants.py b/pilot/util/constants.py index a329c818c..1fdaab8dc 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -14,7 +14,7 @@ RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates REVISION = '1' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '5' # build number should be reset to '1' for every new development cycle +BUILD = '6' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 From a14a41ee9c50eba9d9cf531fc83d71d0985a2c6a Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Mon, 24 Jan 2022 12:04:20 +0100 Subject: [PATCH 08/14] Renamed --verifyproxy to --noproxyverification (previousy misleading name) --- PILOTVERSION | 2 +- pilot.py | 4 ++-- pilot/util/constants.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/PILOTVERSION b/PILOTVERSION index 861a59a15..30a98589e 100644 --- a/PILOTVERSION +++ b/PILOTVERSION @@ -1 +1 @@ -3.1.1.6 \ No newline at end of file +3.1.1.7 \ No newline at end of file diff --git a/pilot.py b/pilot.py index f774aad07..e99ccfd4f 100755 --- a/pilot.py +++ b/pilot.py @@ -7,7 +7,7 @@ # Authors: # - Mario Lassnig, mario.lassnig@cern.ch, 2016-2017 # - Daniel Drizhuk, d.drizhuk@gmail.com, 2017 -# - Paul Nilsson, paul.nilsson@cern.ch, 2017-2021 +# - Paul Nilsson, paul.nilsson@cern.ch, 2017-2022 from __future__ import print_function # Python 2 (2to3 complains about this) from __future__ import absolute_import @@ -196,7 +196,7 @@ def get_args(): default=True, help='Disable server updates') - arg_parser.add_argument('-t', '--verifyproxy', + arg_parser.add_argument('-t', '--noproxyverification', dest='verify_proxy', action='store_false', default=True, diff --git a/pilot/util/constants.py b/pilot/util/constants.py index 1fdaab8dc..10d8960e3 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -14,7 +14,7 @@ RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates REVISION = '1' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '6' # build number should be reset to '1' for every new development cycle +BUILD = '7' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 From 2019c4ffbba4a732f4bce68fb5f5b1d87e9932ad Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Tue, 25 Jan 2022 09:06:43 +0100 Subject: [PATCH 09/14] Fixed error message for zero size output files --- PILOTVERSION | 2 +- pilot/util/constants.py | 2 +- pilot/util/monitoring.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/PILOTVERSION b/PILOTVERSION index 30a98589e..bb994f939 100644 --- a/PILOTVERSION +++ b/PILOTVERSION @@ -1 +1 @@ -3.1.1.7 \ No newline at end of file +3.1.1.8 \ No newline at end of file diff --git a/pilot/util/constants.py b/pilot/util/constants.py index 10d8960e3..27a93f19f 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -14,7 +14,7 @@ RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates REVISION = '1' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '7' # build number should be reset to '1' for every new development cycle +BUILD = '8' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 diff --git a/pilot/util/monitoring.py b/pilot/util/monitoring.py index 436672a16..e4ffe6d57 100644 --- a/pilot/util/monitoring.py +++ b/pilot/util/monitoring.py @@ -631,7 +631,7 @@ def get_max_allowed_work_dir_size(): # grace margin, as discussed in https://its.cern.ch/jira/browse/ATLASPANDA-482 margin = 10.0 # percent, read later from somewhere maxwdirsize = int(maxwdirsize * (1 + margin / 100.0)) - logger.info(f"work directory size check will use {maxwdirsize} B as a max limit (10%% grace limit added)") + logger.info(f"work directory size check will use {maxwdirsize} B as a max limit (10% grace limit added)") return maxwdirsize @@ -692,6 +692,10 @@ def check_output_file_sizes(job): max_fsize = human2bytes(config.Pilot.maximum_output_file_size) if fsize and fsize < max_fsize: logger.info(f'output file {path} is within allowed size limit ({fsize} B < {max_fsize} B)') + elif fsize == 0: + exit_code = errors.EMPTYOUTPUTFILE + diagnostics = f'zero size output file detected: {path}' + logger.warning(diagnostics) else: exit_code = errors.OUTPUTFILETOOLARGE diagnostics = f'output file {path} is not within allowed size limit ({fsize} B > {max_fsize} B)' From 4f67b63f4b7a8f95c7739d0105072b1636f0e94d Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Tue, 25 Jan 2022 09:14:26 +0100 Subject: [PATCH 10/14] Flake8 corrections --- PILOTVERSION | 2 +- pilot/api/analytics.py | 3 +-- pilot/user/atlas/proxy.py | 2 +- pilot/util/constants.py | 2 +- pilot/util/monitoring.py | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/PILOTVERSION b/PILOTVERSION index bb994f939..3a154d4ce 100644 --- a/PILOTVERSION +++ b/PILOTVERSION @@ -1 +1 @@ -3.1.1.8 \ No newline at end of file +3.1.1.9 \ No newline at end of file diff --git a/pilot/api/analytics.py b/pilot/api/analytics.py index b4cd350df..34f1252e8 100644 --- a/pilot/api/analytics.py +++ b/pilot/api/analytics.py @@ -5,7 +5,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 # # Authors: -# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2021 +# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2022 from .services import Services from pilot.common.exception import NotDefined, NotSameLength, UnknownException @@ -210,7 +210,6 @@ def find_limit(self, _x, _y, _chi2_org, norg, change_limit=0.25, edge="right", s _y = _y[steps:] try: fit = self.fit(_x, _y) - _slope = self.slope() except Exception as exc: logger.warning(f'caught exception: {exc}') break diff --git a/pilot/user/atlas/proxy.py b/pilot/user/atlas/proxy.py index 2f736b853..2aec15b72 100644 --- a/pilot/user/atlas/proxy.py +++ b/pilot/user/atlas/proxy.py @@ -13,7 +13,7 @@ import os import logging -from pilot.user.atlas.setup import get_file_system_root_path +# from pilot.user.atlas.setup import get_file_system_root_path from pilot.util.container import execute from pilot.common.errorcodes import ErrorCodes from time import time diff --git a/pilot/util/constants.py b/pilot/util/constants.py index 27a93f19f..57f1670d8 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -14,7 +14,7 @@ RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates REVISION = '1' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '8' # build number should be reset to '1' for every new development cycle +BUILD = '9' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 diff --git a/pilot/util/monitoring.py b/pilot/util/monitoring.py index e4ffe6d57..057701b13 100644 --- a/pilot/util/monitoring.py +++ b/pilot/util/monitoring.py @@ -5,7 +5,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 # # Authors: -# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2021 +# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2022 # This module contains implementations of job monitoring tasks From 804abc629bc144dc39afbab2be3afc003c9a0a07 Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Tue, 25 Jan 2022 09:54:59 +0100 Subject: [PATCH 11/14] Removed workDir from list of items to check in looping test --- PILOTVERSION | 2 +- pilot/user/atlas/loopingjob_definitions.py | 5 ++++- pilot/util/constants.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/PILOTVERSION b/PILOTVERSION index 3a154d4ce..42ab9ff9b 100644 --- a/PILOTVERSION +++ b/PILOTVERSION @@ -1 +1 @@ -3.1.1.9 \ No newline at end of file +3.1.1.10 \ No newline at end of file diff --git a/pilot/user/atlas/loopingjob_definitions.py b/pilot/user/atlas/loopingjob_definitions.py index 21488a7cf..1a368b15b 100644 --- a/pilot/user/atlas/loopingjob_definitions.py +++ b/pilot/user/atlas/loopingjob_definitions.py @@ -5,7 +5,9 @@ # http://www.apache.org/licenses/LICENSE-2.0 # # Authors: -# - Paul Nilsson, paul.nilsson@cern.ch, 2018 +# - Paul Nilsson, paul.nilsson@cern.ch, 2018-2022 + +from os.path import join def allow_loopingjob_detection(): @@ -34,6 +36,7 @@ def remove_unwanted_files(workdir, files): _files = [] for _file in files: if not (workdir == _file or + _file == join(workdir, 'workDir') or "prmon" in _file or "/pilot" in _file or "/pandawnutil" in _file or diff --git a/pilot/util/constants.py b/pilot/util/constants.py index 57f1670d8..5591745d2 100644 --- a/pilot/util/constants.py +++ b/pilot/util/constants.py @@ -14,7 +14,7 @@ RELEASE = '3' # released number should be fixed at 3 for Pilot 3 VERSION = '1' # version number is '1' for first release, '0' until then, increased for bigger updates REVISION = '1' # revision number should be reset to '0' for every new version release, increased for small updates -BUILD = '9' # build number should be reset to '1' for every new development cycle +BUILD = '10' # build number should be reset to '1' for every new development cycle SUCCESS = 0 FAILURE = 1 From 3f7d0d0b1a89cf58951b50c96108e51e41a60c55 Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Tue, 25 Jan 2022 11:06:10 +0100 Subject: [PATCH 12/14] Disabled real-time logging test --- pilot/control/payload.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pilot/control/payload.py b/pilot/control/payload.py index 243546375..878d62d47 100644 --- a/pilot/control/payload.py +++ b/pilot/control/payload.py @@ -348,8 +348,8 @@ def get_logging_info(realtimelogging, catchall, realtime_logname, realtime_loggi info_dic = {} if 'logging=' not in catchall or not realtimelogging: - logger.debug(f'catchall={catchall}') - logger.debug(f'realtimelogging={realtimelogging}') + #logger.debug(f'catchall={catchall}') + #logger.debug(f'realtimelogging={realtimelogging}') return {} # args handling @@ -424,7 +424,7 @@ def run_realtimelog(queues, traces, args): job.realtimelogging = True # testing - job.realtimelogging = True + # job.realtimelogging = True # reset info_dic if real-time logging is not wanted by the job if not job.realtimelogging: info_dic = None @@ -433,7 +433,7 @@ def run_realtimelog(queues, traces, args): job.infosys.queuedata.catchall, args.realtime_logname, args.realtime_logging_server) if not info_dic and job.realtimelogging else info_dic - logger.debug(f'info_dic={info_dic}') + # logger.debug(f'info_dic={info_dic}') if info_dic: args.use_realtime_logging = True realtime_logger = get_realtime_logger(args, info_dic) From a0c9139bc6463168c19982ec912e35d712dc2955 Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Tue, 25 Jan 2022 11:10:18 +0100 Subject: [PATCH 13/14] Added unit testing for Python 3.10 --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index aba4f6c8f..9005dd93b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -11,7 +11,7 @@ jobs: continue-on-error: true strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] env: FLAKE8_VERSION: "==3.8.4" FLAKE8_CONFIG: ".flake8" From 16c6ac8d979a6a9b4d81f016c1667dd24b2d14ff Mon Sep 17 00:00:00 2001 From: Paul Nilsson Date: Tue, 25 Jan 2022 11:14:57 +0100 Subject: [PATCH 14/14] Removed unit tests for Python 3.10 since GitHub Actions is not ready for it yet (setuptools must be used instead of outdated pyparsing) --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 9005dd93b..aba4f6c8f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -11,7 +11,7 @@ jobs: continue-on-error: true strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.6', '3.7', '3.8', '3.9'] env: FLAKE8_VERSION: "==3.8.4" FLAKE8_CONFIG: ".flake8"