Fix file ownership handling for copied host files.

Co-authored-by: Jörg Vehlow <github@jv-coder.de>

.vscode/settings.json

@@ -10,8 +10,6 @@
         "--disable=W0718"
     ],
     "python.testing.pytestArgs": [
-        "--capture=no",
-        "--log-level=DEBUG",
         "tests"
     ],
     "python.testing.unittestEnabled": false,

docs/root_generator.md

@@ -1 +1,58 @@
 # Root Generator
+
+The _root generator_ is responsible for generating and conifguring a tarball of the root filesystem. The _root generator_ supports the following parameters:
+
+- *config_file*: Path to the yaml configuration file of the root filesystem.
+- *output*: Path to the folder where the generated artifacts are placed.
+- *--no-config*: Skip the configuration step. This flag allows a two-step 
+  build which first creates the tarball containing all selected packages,
+  and then applies the use configuration.
+- *--sysroot*: The sysroot flag allows to add additional packages only required
+  as part of the sysroot for cross-compiling.
+
+The core part of the _root generator_ is implemented in _ebcl/tools/root/root.py_.
+The _main_ function takes care of parsing the command line parameters
+and then runs _create_root_ of the _RootGenerator_ class, and finally runs
+_finalize_ to cleanup temporary artifacts.
+
+The build process implemented in *create_root* executes the following high level steps:
+
+- In case of a sysroot build: Add additional packages to the list of selected packages.
+- Create the root tarball using either _elbe_, _kiwi_ or _debootstrap_.
+- In case of not skipping the configuration: Copy the overlays and run the config scripts.
+- Move the resulting tarball to the output folder.
+
+## Implementation details
+
+### Root tarball generation
+
+TODO: write section
+
+### Root configuration
+
+The root filesystem configuration is shared code between the _root generator_ and the _root configurator_ and is implemented in _ebcl/tools/root/__init__.py_. For configuring the root tarball the following steps are executed:
+
+- Extract the tarball to a temporary folder.
+- Copy the host files to this folder, overwriting existing files if necessary.
+- Execute the configuration scripts in the given environment.
+- Pack the result as tarball.
+
+Copying of the files and running the scripts is common code for all tools and implemented in the _Files_ class contained in _ebcl/common/files.py_.
+
+#### Copy the host files
+
+The host files which shall be overlayed to the root filesystem are defined in the configuration file using the _host_files_ parameters. These configuration is parsed
+using _parse_files_ of _ebcl/common/files.py_. For each file or folder a _source_
+value is required. This source value is interpreted as relative path to the config file.
+Optionally a _destination_, a _mode_, a _uid_ and a _gid_ can be given. These additional
+parameters are evaluted by _copy_files_. If _uid_ and _gid_ is not given, the user id 0,
+and the group id 0 is used, which means _root_ user and group. If no _mode_ is given
+the _mode_ is not modified, i.e. the value is kept for the file.
+
+#### Run the configuration scripts
+
+TODO: write section
+
+## Root configurator
+
+The _root configurator_, which is implemented in _ebcl/tools/root/root_config.py_, is a stripped down version of the _root generator_, which only applies the customer specific configruation on top of an existing tarball.

ebcl/common/files.py

@@ -108,7 +108,7 @@ def copy_files(
             if file_dest:
                 dst = os.path.join(dst, file_dest)
 
-            mode: str = entry.get('mode', '600')
+            mode: str = entry.get('mode', None)
             uid: int = int(entry.get('uid', 0))
             gid: int = int(entry.get('gid', 0))
 
@@ -178,14 +178,14 @@ def copy_file(
 
             if file != target:
                 if fix_ownership:
+                    # Change owner to host user and group.
                     self.fake.run_sudo(
                         f'chown -R {os.getuid()}:{os.getgid()} {file}')
-                    self.fake.run_sudo(f'chmod -R 755 {file}')
 
-                if os.path.isfile(file):
-                    self._run_cmd(
-                        f'mkdir -p {os.path.dirname(target)}',
-                        environment, check=False)
+                # Create target directory if it does not exist.
+                self._run_cmd(
+                    f'mkdir -p {os.path.dirname(target)}',
+                    environment, check=False)
 
                 is_dir = os.path.isdir(file)
                 if is_dir:
@@ -194,6 +194,7 @@ def copy_file(
                     logging.debug('File %s is a file...', file)
 
                 if delete_if_exists and not is_dir:
+                    # Delete the target file or folder if it exists.
                     self._run_cmd(f'rm -rf {target}', environment)
 
                 if move:
@@ -210,6 +211,12 @@ def copy_file(
                     self._run_cmd(f'chown {uid} {target}', environment)
                 if gid:
                     self._run_cmd(f'chown :{gid} {target}', environment)
+
+                if not mode and not move:
+                    # Take over mode from source file.
+                    mode = oct(os.stat(file).st_mode)
+                    mode = mode[-4:]
+
                 if mode:
                     self._run_cmd(f'chmod {mode} {target}', environment)
 

robot_tests/initrd.robot

@@ -31,7 +31,7 @@ Rootfs should be set up
 
 File dummy.txt should be OK
     Should Be Owned By   /root/dummy.txt    0    0
-    Should Have Mode     /root/dummy.txt    600
+    Should Have Mode     /root/dummy.txt    664
 
 File other.txt should be OK
     Should Be Owned By   /root/other.txt    123    456

tests/test_files.py

@@ -81,19 +81,19 @@ def test_copy_file_shell(self):
         )
         assert files
 
-        (out, err, ret) = self.fake.run_cmd(
+        (out, err, ret) = self.fake.run_sudo(
             f'file {self.target_dir}/cp_root_shell')
         assert ret == 0
         assert not err.strip()
         assert out
         assert 'ASCII text' in out
 
-        (out, err, ret) = self.fake.run_cmd(
+        (out, err, ret) = self.fake.run_sudo(
             f'stat -c \'%u %g %a\'  {self.target_dir}/cp_root_shell')
         assert ret == 0
         assert not err.strip()
         assert out
-        (mode, _err, _ret) = self.fake.run_cmd(
+        (mode, _err, _ret) = self.fake.run_sudo(
             f'stat -c \'%a\'  {self.other_dir}/root')
         assert mode
         mode = mode.strip()

tests/test_initrd.py

@@ -168,7 +168,9 @@ def test_copy_files(self):
             f'stat -c \'%a\' {self.generator.target_dir}/root/dummy.txt')
         assert out is not None
         out = out.split('\n')[-2]
-        assert out.strip() == '600'
+        mode = oct(
+            os.stat(f'{os.path.dirname(__file__)}/data/dummy.txt').st_mode)
+        assert out.strip() == mode[-3:]
         assert not err.strip()
 
         (out, err, _returncode) = self.fake.run_sudo(

tests/test_root.py

@@ -55,6 +55,7 @@ def test_build_kiwi_image(self):
         assert archive
         assert os.path.isfile(archive)
 
+    @pytest.mark.skip(reason="Elbe is not part of dev container anymore.")
     @pytest.mark.dev_container
     def test_build_root_archive(self):
         """ Test build root.tar. """
@@ -107,6 +108,7 @@ def test_build_sysroot_kiwi(self):
         assert archive
         assert os.path.isfile(archive)
 
+    @pytest.mark.skip(reason="Elbe is not part of dev container anymore.")
     @pytest.mark.dev_container
     def test_build_sysroot_elbe(self):
         """ Test build root.tar. """