Log file created by default for uploads and downloads

SPRINTLOG.md

@@ -418,3 +418,7 @@ _Empty sprint_
 - Re-introduce executable for ubuntu-20.04 ([#744](https://github.com/ScilifelabDataCentre/dds_cli/pull/744))
 - Update upload-artifact action to v4 ([#743](https://github.com/ScilifelabDataCentre/dds_cli/pull/743))
 - New version: 2.10.1 ([#746](https://github.com/ScilifelabDataCentre/dds_cli/pull/746))
+
+# 2025-04-14 - 2025-04-25
+
+- Generate log file by default for put and get ([#737](https://github.com/ScilifelabDataCentre/dds_cli/pull/737))

dds_cli/__init__.py

@@ -181,7 +181,6 @@ class FileSegment:
 # Determine if the user is on an old terminal without proper Unicode support
 dds_on_legacy_console = rich.console.detect_legacy_windows()
 
-
 # Required to make the standalone executables build with PyInstaller work.
 if __name__ == "__main__":
     from dds_cli.__main__ import dds_main

dds_cli/__main__.py

@@ -107,7 +107,19 @@
 @click.option(
     "-v", "--verbose", is_flag=True, default=False, help="Print verbose output to the console."
 )
-@click.option("-l", "--log-file", help="Save a log to a file.", metavar="<filename>")
+@click.option("--force-no-log", help="[NOT RECOMMENDED] Do not save logs to a file.", is_flag=True)
+@click.option(
+    "-l",
+    "--log-file",
+    help=(
+        "Save logs to file. Define a custom path with this option. "
+        "Recommended format: <command>_<date>_<time>.log."
+        "In the case of opening a support ticket regarding the DDS, attach this file. "
+        "Note that a log file will be generated by default for uploads and downloads."
+    ),
+    metavar="<filename>",
+    required=False,
+)
 @click.option(
     "--no-prompt", is_flag=True, default=False, help="Run without any interactive features."
 )
@@ -121,7 +133,7 @@
     help="List the options of any DDS subcommand and its default settings.",
 )
 @click.pass_context
-def dds_main(click_ctx, verbose, log_file, no_prompt, token_path):
+def dds_main(click_ctx, verbose, force_no_log, log_file, no_prompt, token_path):
     """SciLifeLab Data Delivery System (DDS) command line interface.
 
     Access token is saved in a .dds_cli_token file in the home directory.
@@ -139,6 +151,14 @@ def dds_main(click_ctx, verbose, log_file, no_prompt, token_path):
         )
 
     if "--help" not in sys.argv:
+        # Create context object and save command to context
+        click_ctx.obj = {
+            "NO_PROMPT": no_prompt,
+            "TOKEN_PATH": token_path,
+            "COMMAND": sys.argv,
+            "DEFAULT_LOG": True,
+        }
+
         # Set the base logger to output DEBUG
         LOG.setLevel(logging.DEBUG)
 
@@ -153,20 +173,24 @@ def dds_main(click_ctx, verbose, log_file, no_prompt, token_path):
             )
         )
 
-        # Set up logs to a file if we asked for one
         if log_file:
-            log_fh = logging.FileHandler(log_file, encoding="utf-8")
-            log_fh.setLevel(logging.DEBUG)
-            log_fh.setFormatter(
-                logging.Formatter(
-                    fmt="[%(asctime)s] %(name)-15s %(lineno)-5s [%(levelname)-7s]  %(message)s",
-                    datefmt="%Y-%m-%d %H:%M:%S",
-                )
-            )
-            LOG.addHandler(log_fh)
+            # Do not setup log at default location if log_file is specified
+            click_ctx.obj.update({"DEFAULT_LOG": False})
 
-        # Create context object
-        click_ctx.obj = {"NO_PROMPT": no_prompt, "TOKEN_PATH": token_path}
+            if force_no_log:
+                LOG.warning(
+                    "You have used both the '--log-file' and '--force-no-log' option, you can only use one."
+                )
+                sys.exit(1)
+            else:
+                file_handler = dds_cli.utils.setup_logging_to_file(filename=log_file)
+                LOG.addHandler(file_handler)
+        else:
+            if force_no_log:
+                LOG.warning(
+                    "You have chosen to turn off the recommended default logging with the '--force-no-log' option."
+                )
+                click_ctx.obj.update({"DEFAULT_LOG": False})
 
 
 # ************************************************************************************************ #
@@ -1625,9 +1649,37 @@ def put_data(
     delivery to finish. To avoid that a delivery fails because of an expired token, we recommend
     reauthenticating yourself before uploading data.
     """
+    # Define staging directory path
+    staging_dir_path: pathlib.Path = pathlib.Path(
+        f"DataDelivery_{dds_cli.timestamp.TimeStamp().timestamp}_{project}_upload"
+    )
+
+    # Staging directory should either be in specified mount dir or in current location
+    if mount_dir:
+        staging_dir_path = mount_dir / staging_dir_path
+    else:
+        staging_dir_path = pathlib.Path.cwd() / staging_dir_path
+
+    # Generate staging directory
+    staging_dir = dds_cli.directory.DDSDirectory(path=staging_dir_path)
+
+    # Setup logging -- needs to be in this file to work
+    if click_ctx.get("DEFAULT_LOG"):
+        default_log_name = dds_cli.utils.get_default_log_name(
+            command=click_ctx.get("COMMAND", ["commandnotfound"]),
+            log_directory=staging_dir.directories["LOGS"],
+        )
+
+        # Start logging to file
+        file_handler = dds_cli.utils.setup_logging_to_file(filename=default_log_name)
+        LOG.addHandler(file_handler)
+
+    # Log command
+    LOG.debug("Command: %s", " ".join(click_ctx.get("COMMAND")))
+
+    # Run upload
     try:
         dds_cli.data_putter.put(
-            mount_dir=mount_dir,
             project=project,
             source=source,
             source_path_file=source_path_file,
@@ -1638,6 +1690,7 @@ def put_data(
             no_prompt=click_ctx.get("NO_PROMPT", False),
             token_path=click_ctx.get("TOKEN_PATH"),
             destination=destination,
+            staging_dir=staging_dir,
         )
     except (
         dds_cli.exceptions.AuthenticationError,
@@ -1725,6 +1778,30 @@ def get_data(
         )
         sys.exit(1)
 
+    # Define staging directory path
+    staging_dir_path: pathlib.Path = pathlib.Path.cwd() / pathlib.Path(
+        f"DataDelivery_{dds_cli.timestamp.TimeStamp().timestamp}_{project}_download"
+    )
+    if destination:
+        staging_dir_path = destination
+
+    # Generate staging directory
+    staging_dir = dds_cli.directory.DDSDirectory(path=staging_dir_path)
+
+    # Setup logging -- needs to be in this file to work
+    if click_ctx.get("DEFAULT_LOG"):
+        default_log_name = dds_cli.utils.get_default_log_name(
+            command=click_ctx.get("COMMAND", ["commandnotfound"]),
+            log_directory=staging_dir.directories["LOGS"],
+        )
+
+        # Start logging to file
+        file_handler = dds_cli.utils.setup_logging_to_file(filename=default_log_name)
+        LOG.addHandler(file_handler)
+
+    # Log command
+    LOG.debug("Command: %s", " ".join(click_ctx.get("COMMAND")))
+
     try:
         # Begin delivery
         with dds_cli.data_getter.DataGetter(
@@ -1733,11 +1810,11 @@ def get_data(
             source=source,
             source_path_file=source_path_file,
             break_on_fail=break_on_fail,
-            destination=destination,
             silent=silent,
             verify_checksum=verify_checksum,
             no_prompt=click_ctx.get("NO_PROMPT", False),
             token_path=click_ctx.get("TOKEN_PATH"),
+            staging_dir=staging_dir,
         ) as getter:
             with rich.progress.Progress(
                 "{task.description}",

dds_cli/auth.py

@@ -41,7 +41,6 @@ def __init__(
         # Initiate DDSBaseClass to authenticate user
         super().__init__(
             authenticate=authenticate,
-            method_check=False,
             force_renew_token=force_renew_token,
             token_path=token_path,
             totp=totp,

dds_cli/base.py

@@ -18,8 +18,6 @@
 import dds_cli.utils
 
 from dds_cli import (
-    DDS_METHODS,
-    DDS_DIR_REQUIRED_METHODS,
     DDS_KEYS_REQUIRED_METHODS,
 )
 from dds_cli import DDSEndpoint
@@ -45,50 +43,21 @@ class DDSBaseClass:
     def __init__(
         self,
         project=None,
-        dds_directory: pathlib.Path = None,
-        mount_dir: pathlib.Path = None,
         method: str = None,
         authenticate: bool = True,
-        method_check: bool = True,
         force_renew_token: bool = False,
         totp: str = None,
         no_prompt: bool = False,
         token_path: str = None,
         allow_group: bool = False,
+        staging_dir: dds_cli.directory.DDSDirectory = None,
     ):
         """Initialize Base class for authenticating the user and preparing for DDS action."""
         self.project = project
-        self.method_check = method_check
         self.method = method
         self.no_prompt = no_prompt
         self.token_path = token_path
 
-        if self.method_check:
-            # Get attempted operation e.g. put/ls/rm/get
-            if self.method not in DDS_METHODS:
-                raise exceptions.InvalidMethodError(attempted_method=self.method)
-            LOG.debug("Attempted operation: %s", self.method)
-
-            # Use user defined destination if any specified
-            if self.method in DDS_DIR_REQUIRED_METHODS:
-                default_dir = pathlib.Path(
-                    f"DataDelivery_{dds_cli.timestamp.TimeStamp().timestamp}_{self.project}_"
-                    f"{'upload' if self.method == 'put' else 'download'}"
-                )
-                if mount_dir:
-                    new_directory = mount_dir / default_dir
-                elif dds_directory:
-                    new_directory = dds_directory
-                else:
-                    new_directory = pathlib.Path.cwd() / default_dir
-
-                self.temporary_directory = new_directory
-
-                self.dds_directory = dds_cli.directory.DDSDirectory(path=new_directory)
-                self.failed_delivery_log = self.dds_directory.directories["LOGS"] / pathlib.Path(
-                    "dds_failed_delivery.json"
-                )
-
         # Keyboardinterrupt
         self.stop_doing = False
 
@@ -105,7 +74,15 @@ def __init__(
 
         # Project access only required if trying to upload, download or list
         # files within project
+        # TODO: Move to DataPutter / DataGetter??
         if self.method in DDS_KEYS_REQUIRED_METHODS:
+            # NOTE: Might be something to refactor in the future, but needed for now
+            self.dds_directory = staging_dir
+            self.temporary_directory = self.dds_directory.directories["ROOT"]
+            self.failed_delivery_log = self.dds_directory.directories["LOGS"] / pathlib.Path(
+                "dds_failed_delivery.json"
+            )
+
             if self.method == "put":
                 self.s3connector = self.__get_safespring_keys()
 
@@ -234,15 +211,21 @@ def __printout_delivery_summary(self):
         self.filehandler.failed.clear()
 
         if true_failed:
+            log_file_info: str = (
+                "When contacting DDS support, please attach the log file(s) located in "
+                f"{self.dds_directory.directories['LOGS']} to the ticket. "
+                "If you used the '--log-file' option when running your command, "
+                "please also attach that file.\n"
+                "[red][bold]Do not[/bold][/red] delete these files."
+            )
             if self.method == "put":
                 # Raise exception in order to give exit code 1
                 raise exceptions.UploadError(
                     "Errors occurred during upload.\n"
                     "If you wish to retry the upload, re-run the 'dds data put' command again, "
                     "specifying the same options as you did now. To also overwrite the files "
                     "that were uploaded, also add the '--overwrite' flag at the end of the command.\n\n"
-                    f"Please verify that the following error log has been generated: {self.failed_delivery_log}\n"
-                    "[red][bold]Do not[/bold][/red] delete this file; The Data Centre may need it during DDS support."
+                    f"{log_file_info}"
                 )
 
             # TODO: --destination should be able to >at least< overwrite the files in the
@@ -252,7 +235,7 @@ def __printout_delivery_summary(self):
                 "If you wish to retry the download, re-run the `dds data get` command again, "
                 "specifying the same options as you did now. A new directory will "
                 "automatically be created and all files will be downloaded again.\n\n"
-                f"See {self.failed_delivery_log} for more information."
+                f"{log_file_info}"
             )
 
         if nr_uploaded:

dds_cli/data_getter.py

@@ -46,21 +46,21 @@ def __init__(
         get_all: bool = False,
         source: tuple = (),
         source_path_file: pathlib.Path = None,
-        destination: pathlib.Path = pathlib.Path(""),
         silent: bool = False,
         verify_checksum: bool = False,
         method: str = "get",
         no_prompt: bool = False,
         token_path: str = None,
+        staging_dir: dds_cli.directory.DDSDirectory = None,
     ):
         """Handle actions regarding downloading data."""
         # Initiate DDSBaseClass to authenticate user
         super().__init__(
             project=project,
-            dds_directory=destination,
             method=method,
             no_prompt=no_prompt,
             token_path=token_path,
+            staging_dir=staging_dir,
         )
 
         # Initiate DataGetter specific attributes

dds_cli/data_putter.py

@@ -29,6 +29,7 @@
 from dds_cli.custom_decorators import verify_proceed, update_status, subpath_required
 
 import dds_cli
+import dds_cli.directory
 import dds_cli.utils
 
 ###############################################################################
@@ -43,7 +44,6 @@
 
 
 def put(
-    mount_dir,
     project,
     source,
     source_path_file,
@@ -54,11 +54,11 @@ def put(
     no_prompt,
     token_path,
     destination,
+    staging_dir,
 ):
     """Handle upload of data."""
     # Initialize delivery - check user access etc
     with DataPutter(
-        mount_dir=mount_dir,
         project=project,
         source=source,
         source_path_file=source_path_file,
@@ -68,7 +68,9 @@ def put(
         no_prompt=no_prompt,
         token_path=token_path,
         destination=destination,
+        staging_dir=staging_dir,
     ) as putter:
+
         # Progress object to keep track of progress tasks
         with Progress(
             "{task.description}",
@@ -201,7 +203,7 @@ class DataPutter(base.DDSBaseClass):
     def __init__(
         self,
         project: str = None,
-        mount_dir: pathlib.Path = None,
+        staging_dir: dds_cli.directory.DDSDirectory = None,
         break_on_fail: bool = False,
         overwrite: bool = False,
         source: tuple = (),
@@ -216,10 +218,10 @@ def __init__(
         # Initiate DDSBaseClass to authenticate user
         super().__init__(
             project=project,
-            mount_dir=mount_dir,
             method=method,
             no_prompt=no_prompt,
             token_path=token_path,
+            staging_dir=staging_dir,
         )
 
         # Initiate DataPutter specific attributes