Merge pull request #57 from sdss/symlink

Adds follow_symlinks options to commit

CHANGELOG.rst

@@ -11,6 +11,7 @@ This document records the main changes to the sdss_access code.
 - Fix issue `52` - rsync failure when remote file is compressed compared to template
 - Issue `48` - Add support for adding temporary paths for use in local sdss_access
 - Issue `55` - Add support for adding resolved urls or filepaths into sdss_access remote download
+- PR `57` - Enables symlink following by default. Adds a ``follow_symlinks`` option to the ``commit`` method.
 
 3.0.3 (11-29-2023)
 ------------------

docs/sphinx/intro.rst

@@ -307,6 +307,23 @@ The ``input_type`` keyword specifies the type of path input.
     path = '/Users/Brian/Work/sdss/sas/dr17/manga/spectro/redux/v3_1_1/8485/stack/manga-8485-1902-LOGCUBE.fits.gz'
     rsync.add_file(path, input_type='filepath')
 
+Following Symlinks
+^^^^^^^^^^^^^^^^^^
+
+By default ``sdss_access`` will follow symlinks when downloading with rsync or curl.  This behaviour may result in a
+different directory structure from the SAS, and/or duplicated data downloads. To ensure an exact SAS structure, you
+can disable this behaviour by setting the ``follow_symlinks`` flag to False.
+::
+
+    from sdss_access import RsyncAccess
+    rsync = RsyncAccess(release='DR17')
+    rsync.remote()
+    rsync.add('mangacube', drpver='v3_1_1', plate='8485', ifu='*', wave='LOG')
+    rsync.set_stream()
+
+    # disable follow_symlinks
+    rsync.commit(follow_symlinks=False)
+
 
 Accessing SDSS-V Products
 -------------------------

python/sdss_access/sync/baseaccess.py

@@ -223,10 +223,10 @@ def set_stream_task(self, task=None, out=None):
     def _get_stream_command(self):
         ''' gets the stream command used when committing the download '''
 
-    def commit(self, offset=None, limit=None):
+    def commit(self, offset=None, limit=None, follow_symlinks: bool = True):
         """ Start the download """
 
-        self.stream.command = self._get_stream_command()
+        self.stream.command = self._get_stream_command(follow_symlinks=follow_symlinks)
         self.stream.sas_module = self._get_sas_module()
         self.stream.append_tasks_to_streamlets(offset=offset, limit=limit)
         self.stream.commit_streamlets()

python/sdss_access/sync/curl.py

@@ -182,9 +182,10 @@ def _get_sas_module(self):
         ''' gets the sas module used when committing the download '''
         return "sas"
 
-    def _get_stream_command(self):
+    def _get_stream_command(self, follow_symlinks: bool = True):
         ''' gets the stream command used when committing the download '''
         auth = ''
         if self.auth.username and self.auth.password:
             auth = '-u {0}:{1}'.format(self.auth.username, self.auth.password)
-        return "curl {0} --create-dirs --fail -sSRLK {{path}}".format(auth)
+        opts = f"-sSRK{'L' if follow_symlinks else ''}"
+        return "curl {0} --create-dirs --fail {1} {{path}}".format(auth, opts)

python/sdss_access/sync/rsync.py

@@ -68,6 +68,7 @@ def _get_sas_module(self):
         else: sas_module = None
         return sas_module
 
-    def _get_stream_command(self):
+    def _get_stream_command(self, follow_symlinks: bool = True):
         ''' gets the stream command used when committing the download '''
-        return "rsync -avRK --files-from={path} {source}/{sas_module} {destination}{sas_module}/"
+        base = f"rsync -avRK{'L' if follow_symlinks else ''}"
+        return base + " --files-from={path} {source}/{sas_module} {destination}{sas_module}/"

readthedocs.yml

@@ -1,10 +1,12 @@
 version: 2
 
 build:
-   image: latest
+  os: ubuntu-20.04
+  tools:
+    python: '3.9'
 
 python:
-   version: 3.7
+   version: 3.9
    install:
       - method: pip
         path: .

tests/sync/test_curl.py

@@ -42,3 +42,12 @@ def test_add_files(self, curl, expdata, input_type):
         curl.add_file(path, input_type=input_type)
         task = curl.initial_stream.task[0]
         assert task == expout
+
+    @pytest.mark.parametrize('followsym', [True, False])
+    def test_symlink(self, cadd, followsym):
+        """ test the follow symlink option is added or not """
+        cmd = cadd._get_stream_command(follow_symlinks=followsym)
+        if followsym:
+            assert "-sSRKL" in cmd
+        else:
+            assert "-sSRK" in cmd

tests/sync/test_rsync.py

@@ -84,6 +84,15 @@ def test_add_files(self, rsync, expdata, inittask, input_type):
         task = rsync.initial_stream.task[0]
         assert task == inittask[0]
 
+    @pytest.mark.parametrize('followsym', [True, False])
+    def test_symlink(self, radd, followsym):
+        """ test the follow symlink option is added or not """
+        cmd = radd._get_stream_command(follow_symlinks=followsym)
+        if followsym:
+            assert "-avRKL" in cmd
+        else:
+            assert "-avRK" in cmd
+
 
 class TestRsyncFails(object):
 
@@ -103,5 +112,3 @@ def test_initial_stream(self, radd, inittask):
     def test_final_stream(self, rstream, finaltask):
         task = rstream.stream.task
         assert task[0] == finaltask[0]
-
-