Add logging to all biomero scripts by default. Update batch workflows…

… script
NL-BioImaging · Feb 21, 2024 · 3603b65 · 3603b65
1 parent 2befc27
commit 3603b65
Show file tree

Hide file tree

Showing 8 changed files with 361 additions and 104 deletions.
diff --git a/Example_Minimal_Slurm_Script.py b/Example_Minimal_Slurm_Script.py
@@ -15,6 +15,8 @@
 import subprocess
 from biomero import SlurmClient
 import logging
+import os
+import sys
 
 logger = logging.getLogger(__name__)
 
@@ -72,11 +74,11 @@ def runScript():
 
         try:
             scriptParams = client.getInputs(unwrap=True)
-            print(f"Params: {scriptParams}")
-            print(f"Validating slurm connection:\
+            logger.info(f"Params: {scriptParams}")
+            logger.info(f"Validating slurm connection:\
                 {slurmClient.validate()} for {slurmClient.__dict__}")
 
-            print(f"Running py cmd: {scriptParams[_PYCMD]}")
+            logger.info(f"Running py cmd: {scriptParams[_PYCMD]}")
             print_result = []
             cmdlist = []
             if scriptParams[_RUNSLRM]:
@@ -93,10 +95,10 @@ def runScript():
                 # run a list of commands
                 for cmd in cmdlist:
                     results = slurmClient.run(cmd)
-                    print(f"Ran slurm {results}")
+                    logger.info(f"Ran slurm {results}")
             except subprocess.CalledProcessError as e:
                 results = f"Error {e.__dict__}"
-                print(results)
+                logger.info(results)
             finally:
                 print_result.append(f"{results.stdout}")
             client.setOutput("Message", rstring("".join(print_result)))
@@ -105,4 +107,27 @@ def runScript():
 
 
 if __name__ == '__main__':
+    # Some defaults from OMERO; don't feel like reading ice files.
+    # Retrieve the value of the OMERODIR environment variable
+    OMERODIR = os.environ.get('OMERODIR', '/opt/omero/server/OMERO.server')
+    LOGDIR = os.path.join(OMERODIR, 'var', 'log')
+    LOGFORMAT = "%(asctime)s %(levelname)-5.5s [%(name)40s] " \
+                "[%(process)d] (%(threadName)-10s) %(message)s"
+    # Added the process id
+    LOGSIZE = 500000000
+    LOGNUM = 9
+    log_filename = 'biomero.log'
+    # Create a stream handler with INFO level (for OMERO.web output)
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(logging.INFO)
+    # Create DEBUG logging to rotating logfile at var/log
+    logging.basicConfig(level=logging.DEBUG,
+                        format=LOGFORMAT,
+                        handlers=[
+                            stream_handler,
+                            logging.handlers.RotatingFileHandler(
+                                os.path.join(LOGDIR, log_filename),
+                                maxBytes=LOGSIZE,
+                                backupCount=LOGNUM)
+                        ])
     runScript()
diff --git a/data/SLURM_Get_Results.py b/data/SLURM_Get_Results.py
@@ -838,7 +838,27 @@ def runScript():
 
 
 if __name__ == '__main__':
-    logging.basicConfig(level=logging.INFO,
-                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
-                        stream=sys.stdout)
+    # Some defaults from OMERO; don't feel like reading ice files.
+    # Retrieve the value of the OMERODIR environment variable
+    OMERODIR = os.environ.get('OMERODIR', '/opt/omero/server/OMERO.server')
+    LOGDIR = os.path.join(OMERODIR, 'var', 'log')
+    LOGFORMAT = "%(asctime)s %(levelname)-5.5s [%(name)40s] " \
+                "[%(process)d] (%(threadName)-10s) %(message)s"
+    # Added the process id
+    LOGSIZE = 500000000
+    LOGNUM = 9
+    log_filename = 'biomero.log'
+    # Create a stream handler with INFO level (for OMERO.web output)
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(logging.INFO)
+    # Create DEBUG logging to rotating logfile at var/log
+    logging.basicConfig(level=logging.DEBUG,
+                        format=LOGFORMAT,
+                        handlers=[
+                            stream_handler,
+                            logging.handlers.RotatingFileHandler(
+                                os.path.join(LOGDIR, log_filename),
+                                maxBytes=LOGSIZE,
+                                backupCount=LOGNUM)
+                        ])
     runScript()
diff --git a/data/SLURM_Get_Update.py b/data/SLURM_Get_Update.py
@@ -18,6 +18,8 @@
 from omero.rtypes import rstring, robject, unwrap, wrap
 from biomero import SlurmClient
 import logging
+import os
+import sys
 
 logger = logging.getLogger(__name__)
 
@@ -163,4 +165,27 @@ def check_job(slurmClient, message, slurm_job_id):
 
 
 if __name__ == '__main__':
+    # Some defaults from OMERO; don't feel like reading ice files.
+    # Retrieve the value of the OMERODIR environment variable
+    OMERODIR = os.environ.get('OMERODIR', '/opt/omero/server/OMERO.server')
+    LOGDIR = os.path.join(OMERODIR, 'var', 'log')
+    LOGFORMAT = "%(asctime)s %(levelname)-5.5s [%(name)40s] " \
+                "[%(process)d] (%(threadName)-10s) %(message)s"
+    # Added the process id
+    LOGSIZE = 500000000
+    LOGNUM = 9
+    log_filename = 'biomero.log'
+    # Create a stream handler with INFO level (for OMERO.web output)
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(logging.INFO)
+    # Create DEBUG logging to rotating logfile at var/log
+    logging.basicConfig(level=logging.DEBUG,
+                        format=LOGFORMAT,
+                        handlers=[
+                            stream_handler,
+                            logging.handlers.RotatingFileHandler(
+                                os.path.join(LOGDIR, log_filename),
+                                maxBytes=LOGSIZE,
+                                backupCount=LOGNUM)
+                        ])
     runScript()
diff --git a/data/_SLURM_Image_Transfer.py b/data/_SLURM_Image_Transfer.py
@@ -25,7 +25,7 @@
 zip file for download, then exports it to SLURM.
 
 @author Torec Luik
-@version 0.0.4
+@version 1.5.0
 """
 
 import shutil
@@ -48,6 +48,7 @@
     import Image
 from biomero import SlurmClient
 import logging
+import sys
 
 logger = logging.getLogger(__name__)
 
@@ -711,4 +712,27 @@ def run_script():
 
 
 if __name__ == "__main__":
+    # Some defaults from OMERO; don't feel like reading ice files.
+    # Retrieve the value of the OMERODIR environment variable
+    OMERODIR = os.environ.get('OMERODIR', '/opt/omero/server/OMERO.server')
+    LOGDIR = os.path.join(OMERODIR, 'var', 'log')
+    LOGFORMAT = "%(asctime)s %(levelname)-5.5s [%(name)40s] " \
+                "[%(process)d] (%(threadName)-10s) %(message)s"
+    # Added the process id
+    LOGSIZE = 500000000
+    LOGNUM = 9
+    log_filename = 'biomero.log'
+    # Create a stream handler with INFO level (for OMERO.web output)
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(logging.INFO)
+    # Create DEBUG logging to rotating logfile at var/log
+    logging.basicConfig(level=logging.DEBUG,
+                        format=LOGFORMAT,
+                        handlers=[
+                            stream_handler,
+                            logging.handlers.RotatingFileHandler(
+                                os.path.join(LOGDIR, log_filename),
+                                maxBytes=LOGSIZE,
+                                backupCount=LOGNUM)
+                        ])
     run_script()
diff --git a/init/SLURM_Init_environment.py b/init/SLURM_Init_environment.py
@@ -15,6 +15,8 @@
 from omero.rtypes import rstring, unwrap
 from biomero import SlurmClient
 import logging
+import os
+import sys
 
 logger = logging.getLogger(__name__)
 
@@ -61,4 +63,27 @@ def runScript():
 
 
 if __name__ == '__main__':
+    # Some defaults from OMERO; don't feel like reading ice files.
+    # Retrieve the value of the OMERODIR environment variable
+    OMERODIR = os.environ.get('OMERODIR', '/opt/omero/server/OMERO.server')
+    LOGDIR = os.path.join(OMERODIR, 'var', 'log')
+    LOGFORMAT = "%(asctime)s %(levelname)-5.5s [%(name)40s] " \
+                "[%(process)d] (%(threadName)-10s) %(message)s"
+    # Added the process id
+    LOGSIZE = 500000000
+    LOGNUM = 9
+    log_filename = 'biomero.log'
+    # Create a stream handler with INFO level (for OMERO.web output)
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(logging.INFO)
+    # Create DEBUG logging to rotating logfile at var/log
+    logging.basicConfig(level=logging.DEBUG,
+                        format=LOGFORMAT,
+                        handlers=[
+                            stream_handler,
+                            logging.handlers.RotatingFileHandler(
+                                os.path.join(LOGDIR, log_filename),
+                                maxBytes=LOGSIZE,
+                                backupCount=LOGNUM)
+                        ])
     runScript()
diff --git a/workflows/SLURM_CellPose_Segmentation.py b/workflows/SLURM_CellPose_Segmentation.py
@@ -16,6 +16,8 @@
 
 from __future__ import print_function
 import omero
+import os
+import sys
 from omero.grid import JobParams
 from omero.rtypes import rstring, unwrap
 import omero.scripts as omscripts
@@ -38,7 +40,7 @@ def runScript():
 
         params = JobParams()
         params.authors = ["Torec Luik"]
-        params.version = "0.0.4"
+        params.version = "1.5.0"
         params.description = f'''Script to run CellPose on slurm cluster.
         First run the {_IMAGE_EXPORT_SCRIPT} script to export your data
         to the cluster.
@@ -51,7 +53,7 @@ def runScript():
         Connection ready? {slurmClient.validate()}
         '''
         params.name = 'Slurm Cellpose Segmentation'
-        params.contact = 't.t.luik@amsterdamumc.nl'
+        params.contact = 'cellularimaging@amsterdamumc.nl'
         params.institutions = ["Amsterdam UMC"]
         params.authorsInstitutions = [[1]]
 
@@ -174,4 +176,27 @@ def runScript():
 
 
 if __name__ == '__main__':
+    # Some defaults from OMERO; don't feel like reading ice files.
+    # Retrieve the value of the OMERODIR environment variable
+    OMERODIR = os.environ.get('OMERODIR', '/opt/omero/server/OMERO.server')
+    LOGDIR = os.path.join(OMERODIR, 'var', 'log')
+    LOGFORMAT = "%(asctime)s %(levelname)-5.5s [%(name)40s] " \
+                "[%(process)d] (%(threadName)-10s) %(message)s"
+    # Added the process id
+    LOGSIZE = 500000000
+    LOGNUM = 9
+    log_filename = 'biomero.log'
+    # Create a stream handler with INFO level (for OMERO.web output)
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(logging.INFO)
+    # Create DEBUG logging to rotating logfile at var/log
+    logging.basicConfig(level=logging.DEBUG,
+                        format=LOGFORMAT,
+                        handlers=[
+                            stream_handler,
+                            logging.handlers.RotatingFileHandler(
+                                os.path.join(LOGDIR, log_filename),
+                                maxBytes=LOGSIZE,
+                                backupCount=LOGNUM)
+                        ])
     runScript()