diff --git a/.ci/config.cmake b/.ci/config.cmake new file mode 100644 index 00000000000..1da3b58ada5 --- /dev/null +++ b/.ci/config.cmake @@ -0,0 +1,17 @@ +set(FORCE_ENABLED 0 CACHE BOOL "") +set(ENABLE_ASAN 0 CACHE BOOL "") +set(ENABLE_TSAN 0 CACHE BOOL "") +set(ENABLE_FUSE 1 CACHE BOOL "") +set(ENABLE_HTTP 1 CACHE BOOL "") +set(ENABLE_KRB5 1 CACHE BOOL "") +set(ENABLE_MACAROONS 1 CACHE BOOL "") +set(ENABLE_PYTHON 1 CACHE BOOL "") +set(ENABLE_READLINE 1 CACHE BOOL "") +set(ENABLE_SCITOKENS 1 CACHE BOOL "") +set(ENABLE_TESTS 1 CACHE BOOL "") +set(ENABLE_VOMS 1 CACHE BOOL "") +set(ENABLE_XRDCL 1 CACHE BOOL "") +set(ENABLE_XRDCLHTTP 1 CACHE BOOL "") +set(ENABLE_XRDEC 1 CACHE BOOL "") +set(XRDCL_LIB_ONLY 0 CACHE BOOL "") +set(XRDCL_ONLY 0 CACHE BOOL "") diff --git a/.gitattributes b/.gitattributes index 0893fe6afee..5f72683f8fc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -VERSION_INFO export-subst +VERSION export-subst diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 00000000000..00a9de9fab8 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,333 @@ +name: CI + +on: + push: + branches-ignore: + - master + paths-ignore: + - .gitignore + - .gitlab-ci.yml + - .mailmap + - '**.md' + - 'docs/**' + - 'docker/**' + tags-ignore: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + CDASH: ${{ vars.CDASH }} + CMAKE_VERBOSE_MAKEFILE: true + CTEST_OUTPUT_ON_FAILURE: true + +jobs: + alpine: + name: Alpine + runs-on: ubuntu-latest + container: alpine + + env: + CMAKE_ARGS: -DCMAKE_INSTALL_PREFIX=/usr + + steps: + - name: Install dependencies + shell: sh + run: | + apk add \ + bash \ + cmake \ + cppunit-dev \ + ceph-dev \ + curl-dev \ + fuse-dev \ + fuse3-dev \ + g++ \ + git \ + gtest-dev \ + isa-l-dev \ + json-c-dev \ + krb5-dev \ + libxml2-dev \ + linux-headers \ + make \ + openssl \ + openssl-dev \ + py3-pip \ + py3-setuptools \ + py3-wheel \ + python3-dev \ + readline-dev \ + sudo \ + tinyxml-dev \ + util-linux-dev \ + uuidgen \ + zlib-dev + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup GitHub runner user within container + run: adduser -D --uid 1001 runner && chown -R runner:runner ${GITHUB_WORKSPACE} + + - name: Build and Test with CTest + run: sudo -E -u runner ctest -VV -S test.cmake + + - name: Install with CMake + run: cmake --install build + + - name: Run post-install tests + run: | + tests/post-install.sh + tests/check-headers.sh + + centos7: + name: CentOS 7 + runs-on: ubuntu-latest + container: centos:7 + + env: + CMAKE_ARGS: "-DCMAKE_INSTALL_PREFIX=/usr;-DCMAKE_INSTALL_RPATH='$ORIGIN/../$LIB'" + + steps: + - name: Install dependencies + run: | + yum install -y centos-release-scl epel-release git + yum install -y epel-rpm-macros rpmdevtools sudo yum-utils + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup GitHub runner user within container + run: adduser --uid 1001 runner && chown -R runner:runner . + + - name: Install XRootD build dependencies + run: yum-builddep -y xrootd.spec + + - name: Build and Test with CTest + run: | + source /opt/rh/devtoolset-7/enable + su -p runner -c 'ctest3 -VV -S test.cmake' + + alma8: + name: Alma 8 + runs-on: ubuntu-latest + container: almalinux:8 + + env: + CMAKE_ARGS: "-DCMAKE_INSTALL_PREFIX=/usr;-DCMAKE_INSTALL_RPATH='$ORIGIN/../$LIB'" + + steps: + - name: Install dependencies + run: | + dnf install -y dnf-plugins-core epel-release git rpmdevtools sudo + dnf config-manager --set-enabled powertools + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup GitHub runner user within container + run: adduser --uid 1001 runner && chown -R runner:runner ${GITHUB_WORKSPACE} + + - name: Install XRootD build dependencies + run: dnf builddep -y xrootd.spec + + - name: Build and Test with CTest + run: sudo -E -u runner ctest -VV -S test.cmake + + - name: Install with CMake + run: cmake --install build + + - name: Run post-install tests + run: | + tests/post-install.sh + tests/check-headers.sh + + alma9: + name: Alma 9 + runs-on: ubuntu-latest + container: almalinux:9 + + env: + CMAKE_ARGS: "-DCMAKE_INSTALL_PREFIX=/usr;-DCMAKE_INSTALL_RPATH='$ORIGIN/../$LIB'" + + steps: + - name: Install dependencies + run: | + dnf install -y dnf-plugins-core epel-release git rpmdevtools sudo + dnf config-manager --set-enabled crb + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup GitHub runner user within container + run: adduser --uid 1001 runner && chown -R runner:runner ${GITHUB_WORKSPACE} + + - name: Install XRootD build dependencies + run: dnf builddep -y xrootd.spec + + - name: Build and Test with CTest + run: sudo -E -u runner ctest -VV -S test.cmake + + - name: Install with CMake + run: cmake --install build + + - name: Run post-install tests + run: | + tests/post-install.sh + tests/check-headers.sh + + fedora: + name: Fedora + runs-on: ubuntu-latest + container: fedora + + env: + CMAKE_GENERATOR: Ninja + CMAKE_ARGS: "-DCMAKE_INSTALL_PREFIX=/usr;-DCMAKE_INSTALL_RPATH='$ORIGIN/../$LIB'" + + steps: + - name: Install dependencies + run: dnf install -y dnf-plugins-core git ninja-build rpmdevtools + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup GitHub runner user within container + run: adduser --uid 1001 runner && chown -R runner:runner ${GITHUB_WORKSPACE} + + - name: Install XRootD build dependencies + run: dnf builddep -y --define 'with_ceph 1' xrootd.spec + + - name: Build and Test with CTest + run: sudo -E -u runner ctest -VV -S test.cmake + + - name: Install with CMake + run: cmake --install build + + - name: Run post-install tests + run: | + tests/post-install.sh + tests/check-headers.sh + + ubuntu: + name: Ubuntu + runs-on: ubuntu-latest + + strategy: + matrix: + compiler: [ gcc, clang ] + + env: + CC: ${{ matrix.compiler }} + DEBIAN_FRONTEND: noninteractive + CMAKE_ARGS: '-DINSTALL_PYTHON_BINDINGS=0;-DUSE_SYSTEM_ISAL=1;-DCMAKE_INSTALL_PREFIX=/usr' + + steps: + - name: Install dependencies + run: | + sudo apt update -q + sudo apt install -y \ + cmake \ + clang \ + davix-dev \ + g++ \ + libcppunit-dev \ + libcurl4-openssl-dev \ + libfuse-dev \ + libgtest-dev \ + libisal-dev \ + libjson-c-dev \ + libkrb5-dev \ + libmacaroons-dev \ + libreadline-dev \ + libscitokens-dev \ + libssl-dev \ + libsystemd-dev \ + libtinyxml-dev \ + libxml2-dev \ + make \ + pkg-config \ + python3-dev \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + uuid-dev \ + voms-dev \ + zlib1g-dev + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build and Test with CTest + run: env CC=${CC} CXX=${CC/g*/g++} ctest -VV -S test.cmake + + - name: Install with CMake + run: sudo cmake --install build + + - name: Install Python bindings + run: | + sudo python3 -m pip install \ + --target /usr/lib/python3/dist-packages \ + --use-pep517 --verbose build/bindings/python + + - name: Run post-install tests + run: | + tests/post-install.sh + tests/check-headers.sh + + macos: + name: macOS + runs-on: macos-latest + + env: + CC: clang + CXX: clang++ + CMAKE_ARGS: "-DPython_FIND_UNVERSIONED_NAMES=FIRST" + CMAKE_PREFIX_PATH: /usr/local/opt/openssl@3 + + steps: + - name: Workaround for issue 1772 + run: sudo sed -i -e "s/localhost/localhost $(hostname)/g" /etc/hosts + + - name: Install dependencies with Homebrew + run: brew install cppunit davix googletest isa-l + + - name: Install Python dependencies with pip + run: python3 -m pip install --upgrade pip setuptools wheel + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build and Test with CTest + run: ctest -VV --repeat until-pass:3 -S test.cmake + + - name: Install with CMake + run: cmake --install build + + - name: Run post-install tests + run: | + export PYVERSION=$(python3 --version | grep -o 3...) + export PYTHONPATH=/usr/local/lib/python${PYVERSION}/site-packages + tests/post-install.sh diff --git a/.github/workflows/DEB.yml b/.github/workflows/DEB.yml new file mode 100644 index 00000000000..4f6c7a34f41 --- /dev/null +++ b/.github/workflows/DEB.yml @@ -0,0 +1,121 @@ +name: DEB + +on: + push: + branches: + - master + paths-ignore: + - .gitignore + - .gitlab-ci.yml + - .mailmap + - '**.md' + - 'docs/**' + - 'docker/**' + tags-ignore: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + DEBIAN_FRONTEND: noninteractive + +jobs: + debian: + name: Debian + + strategy: + matrix: + version: [ 11, 12 ] + + runs-on: ubuntu-latest + container: debian:${{ matrix.version }} + + steps: + - name: Install development tools + run: | + apt update -qq + apt install -y build-essential devscripts git + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install XRootD build dependencies + run: mk-build-deps --install --remove debian/control <<< yes + + - name: Build DEBs + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + VERSION=$(git describe --match 'v*' | sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./') + dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' + debuild --no-tgz-check --no-sign -b + + - name: Install DEBs + run: apt install -y ../*.deb + + - name: Run post-install tests + run: tests/post-install.sh + + - name: Move DEBs to Artifact Directory + run: | + source /etc/os-release + mkdir -p DEB/${ID}/${VERSION_CODENAME} + mv ../*.* DEB/${ID}/${VERSION_CODENAME} + + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: DEB + path: DEB + retention-days: 1 + + ubuntu: + name: Ubuntu (22.04) + runs-on: ubuntu-22.04 + + steps: + - name: Install development tools + run: | + sudo apt update -qq + sudo apt install -y build-essential devscripts equivs git + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install XRootD build dependencies + run: mk-build-deps --install --remove -s sudo debian/control <<< yes + + - name: Build DEBs + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + VERSION=$(git describe --match 'v*' | sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./') + dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' + debuild --no-tgz-check --no-sign -b + + - name: Install DEBs + run: sudo apt install -y ../*.deb + + - name: Run post-install tests + run: tests/post-install.sh + + - name: Move DEBs to Artifact Directory + run: | + source /etc/os-release + mkdir -p DEB/${ID}/${VERSION_CODENAME} + mv ../*.* DEB/${ID}/${VERSION_CODENAME} + + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: DEB + path: DEB + retention-days: 1 diff --git a/.github/workflows/QEMU.yml b/.github/workflows/QEMU.yml new file mode 100644 index 00000000000..ae9534570da --- /dev/null +++ b/.github/workflows/QEMU.yml @@ -0,0 +1,57 @@ +name: QEMU + +on: + workflow_dispatch: + inputs: + os: + description: 'OS' + required: true + default: 'fedora' + type: choice + options: + - alma8 + - alma9 + - centos7 + - debian + - fedora + - ubuntu + arch: + description: 'Architecture' + required: true + default: 's390x' + type: choice + options: + - 386 + - amd64 + - arm + - arm64 + - ppc64le + - s390x + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.os }}-${{ inputs.arch }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +env: + DOCKER: podman + +jobs: + buildx: + name: QEMU (${{ inputs.os }}-${{ inputs.arch }}) + runs-on: ubuntu-latest + + steps: + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Setup QEMU for cross-building images + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Cross-build container with docker/podman buildx + run: cd docker && ./xrd-docker buildx ${{ inputs.os }} ${{ inputs.arch }} diff --git a/.github/workflows/RPM.yml b/.github/workflows/RPM.yml new file mode 100644 index 00000000000..423a1a4b939 --- /dev/null +++ b/.github/workflows/RPM.yml @@ -0,0 +1,197 @@ +name: RPM + +on: + push: + branches: + - master + paths-ignore: + - .gitignore + - .gitlab-ci.yml + - .mailmap + - '**.md' + - 'docs/**' + - 'docker/**' + tags-ignore: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + centos7: + name: CentOS 7 + runs-on: ubuntu-latest + container: centos:7 + + steps: + - name: Install git + run: yum install -y git + + - name: Clone repository + uses: actions/checkout@v1 + + - name: Install RPM development tools + run: | + yum install -y centos-release-scl epel-release + yum install -y epel-rpm-macros rpmdevtools yum-utils + + - name: Install XRootD build dependencies + run: yum-builddep -y xrootd.spec + + - name: Build RPMs + run: | + rpmdev-setuptree + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD + rpmbuild -bb --with git xrootd.spec + + - name: Install RPMs + run: yum install -y $(rpm -E '%{_rpmdir}')/*/*.rpm + + - name: Run post-install tests + run: tests/post-install.sh + + - name: Move RPMs to Artifact Directory + run: mkdir RPMS && mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) + + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: RPM + path: RPMS + retention-days: 1 + + alma8: + name: Alma Linux 8 + runs-on: ubuntu-latest + container: almalinux:8 + + steps: + - name: Install git + run: yum install -y git + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install RPM development tools + run: | + dnf install -y epel-release rpmdevtools dnf-plugins-core + dnf config-manager --set-enabled powertools + + - name: Install XRootD build dependencies + run: dnf builddep -y xrootd.spec + + - name: Build RPMs + run: | + rpmdev-setuptree + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD + rpmbuild -bb --with git xrootd.spec + + - name: Install RPMs + run: dnf install -y $(rpm -E '%{_rpmdir}')/*/*.rpm + + - name: Run post-install tests + run: tests/post-install.sh + + - name: Move RPMs to Artifact Directory + run: mkdir RPMS && mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) + + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: RPM + path: RPMS + retention-days: 1 + + alma9: + name: Alma Linux 9 + runs-on: ubuntu-latest + container: almalinux:9 + + steps: + - name: Install git + run: yum install -y git + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install RPM development tools + run: | + dnf install -y epel-release rpmdevtools dnf-plugins-core + dnf config-manager --set-enabled crb + + - name: Install XRootD build dependencies + run: dnf builddep -y xrootd.spec + + - name: Build RPMs + run: | + rpmdev-setuptree + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD + rpmbuild -bb --with git xrootd.spec + + - name: Install RPMs + run: dnf install -y $(rpm -E '%{_rpmdir}')/*/*.rpm + + - name: Run post-install tests + run: tests/post-install.sh + + - name: Move RPMs to Artifact Directory + run: mkdir RPMS && mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) + + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: RPM + path: RPMS + retention-days: 1 + + fedora: + name: Fedora 39 + runs-on: ubuntu-latest + container: fedora:39 + + steps: + - name: Install git + run: yum install -y git + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install RPM development tools + run: | + dnf install -y rpmdevtools dnf-plugins-core + + - name: Install XRootD build dependencies + run: dnf builddep -y --define 'with_ceph 1' xrootd.spec + + - name: Build RPMs + run: | + rpmdev-setuptree + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD + rpmbuild -bb --with git --with ceph xrootd.spec + + - name: Install RPMs + run: dnf install -y $(rpm -E '%{_rpmdir}')/*/*.rpm + + - name: Run post-install tests + run: tests/post-install.sh + + - name: Move RPMs to Artifact Directory + run: mkdir RPMS && mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) + + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: RPM + path: RPMS + retention-days: 1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index abd2a4ba256..00000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,987 +0,0 @@ -name: build - -on: - push: - pull_request: - schedule: - - cron: '23 1 * * 0' - release: - types: [published] - workflow_dispatch: - -defaults: - run: - shell: bash - -concurrency: - group: build-${{ github.ref }} - cancel-in-progress: true - -jobs: - - cmake-almalinux8: - - runs-on: ubuntu-latest - container: almalinux:8 - - steps: - - name: Install external dependencies with yum - run: | - dnf update -y - dnf clean all - dnf install -y epel-release - dnf install -y --enablerepo=powertools \ - cmake \ - cppunit-devel \ - curl-devel \ - davix-devel \ - diffutils \ - file \ - fuse-devel \ - gcc-c++ \ - git \ - gtest-devel \ - json-c-devel \ - krb5-devel \ - libmacaroons-devel \ - libtool \ - libuuid-devel \ - libxml2-devel \ - make \ - openssl-devel \ - python3-devel \ - python3-setuptools \ - readline-devel \ - scitokens-cpp-devel \ - systemd-devel \ - tinyxml-devel \ - voms-devel \ - yasm \ - zlib-devel - dnf clean all - - - name: Clone repository - uses: actions/checkout@v3 - - - name: Build with cmake - run: | - cd .. - cmake \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DPython_EXECUTABLE=$(command -v python3) \ - -DFORCE_ENABLED=ON \ - -DENABLE_TESTS=ON \ - -DENABLE_XRDEC=ON \ - -DENABLE_MACAROONS=ON \ - -DENABLE_SCITOKENS=ON \ - -DPIP_OPTIONS="--verbose" \ - -S xrootd \ - -B build - cmake build -LH - cmake \ - --build build \ - --clean-first \ - --parallel $(($(nproc) - 1)) - cmake --build build --target install - python3 -m pip list - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - - name: Run tests with CTest - run: | - ctest --output-on-failure --test-dir ../build - - cmake-almalinux9: - - runs-on: ubuntu-latest - container: almalinux:9 - - steps: - - name: Install external dependencies with yum - run: | - dnf update -y - dnf clean all - dnf install -y epel-release - dnf install -y --enablerepo=crb \ - cmake \ - cppunit-devel \ - curl-devel \ - davix-devel \ - diffutils \ - file \ - fuse-devel \ - gcc-c++ \ - git \ - gtest-devel \ - json-c-devel \ - krb5-devel \ - libmacaroons-devel \ - libtool \ - libuuid-devel \ - libxml2-devel \ - make \ - openssl-devel \ - python3-devel \ - python3-setuptools \ - readline-devel \ - scitokens-cpp-devel \ - systemd-devel \ - tinyxml-devel \ - voms-devel \ - yasm \ - zlib-devel - dnf clean all - - - name: Clone repository - uses: actions/checkout@v3 - - - name: Build with cmake - run: | - cd .. - cmake \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DPython_EXECUTABLE=$(command -v python3) \ - -DFORCE_ENABLED=ON \ - -DENABLE_TESTS=ON \ - -DENABLE_XRDEC=ON \ - -DENABLE_MACAROONS=ON \ - -DENABLE_SCITOKENS=ON \ - -DPIP_OPTIONS="--verbose" \ - -S xrootd \ - -B build - cmake build -LH - cmake \ - --build build \ - --clean-first \ - --parallel $(($(nproc) - 1)) - cmake --build build --target install - python3 -m pip list - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - - name: Run tests with CTest - run: | - ctest --output-on-failure --test-dir ../build - - cmake-alpine-musl: - - runs-on: ubuntu-latest - container: alpine - - steps: - - name: Install external dependencies - shell: sh - run: | - apk add \ - bash \ - cmake \ - cppunit-dev \ - curl-dev \ - fuse-dev \ - fuse3-dev \ - g++ \ - git \ - gtest-dev \ - json-c-dev \ - krb5-dev \ - libxml2-dev \ - linux-headers \ - make \ - openssl-dev \ - py3-pip \ - python3-dev \ - readline-dev \ - tinyxml-dev \ - util-linux-dev \ - zlib-dev - - - name: Clone repository - uses: actions/checkout@v3 - - - name: Build with cmake - run: | - cd .. - # need to fix ownership not to confuse git - chown -R -v "$( id -u; ):$( id -g; )" xrootd - cmake \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DPYTHON_EXECUTABLE=$(command -v python3) \ - -DFORCE_ENABLED=ON \ - -DENABLE_HTTP=OFF \ - -DENABLE_TESTS=ON \ - -DENABLE_VOMS=OFF \ - -DENABLE_XRDEC=OFF \ - -DENABLE_XRDCLHTTP=OFF \ - -DENABLE_MACAROONS=OFF \ - -DENABLE_SCITOKENS=OFF \ - -DPIP_OPTIONS="--verbose" \ - -S xrootd \ - -B build - cmake build -LH - cmake \ - --build build \ - --clean-first \ - --parallel $(($(nproc) - 1)) - cmake --build build --target install - python3 -m pip list - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - - name: Run tests with CTest - run: | - ctest --output-on-failure --test-dir ../build - - cmake-centos7: - - runs-on: ubuntu-latest - container: centos:7 - - steps: - - name: Install external dependencies with yum - run: | - yum update -y - yum install -y epel-release centos-release-scl - yum clean all - yum install --nogpg -y \ - cmake3 \ - make \ - krb5-devel \ - libuuid-devel \ - libxml2-devel \ - openssl-devel \ - systemd-devel \ - zlib-devel \ - devtoolset-7-gcc-c++ \ - python3-devel \ - python3-setuptools \ - git \ - cppunit-devel \ - gtest-devel - yum clean all - - # Need to use v1 of action as image Git is too old - - name: Clone repository now that Git is available - uses: actions/checkout@v1 - - - name: Build with cmake - run: | - . /opt/rh/devtoolset-7/enable - cd .. - cmake3 \ - -DCMAKE_INSTALL_PREFIX=/usr/local/ \ - -DPython_EXECUTABLE=$(command -v python3) \ - -DENABLE_TESTS=ON \ - -DPIP_OPTIONS="--verbose" \ - -S xrootd \ - -B build - cmake3 build -LH - cmake3 \ - --build build \ - --clean-first \ - --parallel $(($(nproc) - 1)) - cmake3 --build build --target install - python3 -m pip list - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - - name: Run tests with CTest - run: | - cd ../build - ctest3 --output-on-failure - - cmake-centos7-updated-python: - - runs-on: ubuntu-latest - container: centos:7 - - steps: - - name: Install external dependencies with yum - run: | - yum update -y - yum install -y epel-release centos-release-scl - yum clean all - yum install --nogpg -y \ - cmake3 \ - make \ - krb5-devel \ - libuuid-devel \ - libxml2-devel \ - openssl-devel \ - systemd-devel \ - zlib-devel \ - devtoolset-7-gcc-c++ \ - python3-devel \ - python3-setuptools \ - git \ - cppunit-devel \ - gtest-devel - yum clean all - python3 -m pip --no-cache-dir install --upgrade pip setuptools wheel - - # Need to use v1 of action as image Git is too old - - name: Clone repository now that Git is available - uses: actions/checkout@v1 - - # Use extra PIP_OPTIONS strings as example that this is possible. - # N.B.: None of the PIP_OPTIONS are required for this step to work. - - name: Build with cmake - run: | - . /opt/rh/devtoolset-7/enable - cd .. - cmake3 \ - -DCMAKE_INSTALL_PREFIX=/usr/local/ \ - -DPython_EXECUTABLE=$(command -v python3) \ - -DENABLE_TESTS=ON \ - -DPIP_OPTIONS="--verbose --force-reinstall --prefix /usr/local/" \ - -S xrootd \ - -B build - cmake3 build -LH - cmake3 \ - --build build \ - --clean-first \ - --parallel $(($(nproc) - 1)) - cmake3 --build build --target install - python3 -m pip list - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - # TODO: Drop once Python 2 support is dropped - cmake-centos7-python2: - - runs-on: ubuntu-latest - container: centos:7 - - steps: - # python2-pip is broken on CentOS so can't upgrade pip, setuptools, or wheel - - name: Install external dependencies with yum - run: | - yum update -y - yum install -y epel-release centos-release-scl - yum clean all - yum install --nogpg -y \ - cmake3 \ - make \ - krb5-devel \ - libuuid-devel \ - libxml2-devel \ - openssl-devel \ - systemd-devel \ - zlib-devel \ - devtoolset-7-gcc-c++ \ - python2-pip \ - python2-setuptools \ - python2-devel \ - git \ - cppunit-devel \ - gtest-devel - yum clean all - - # Need to use v1 of action as image Git is too old - - name: Clone repository now that Git is available - uses: actions/checkout@v1 - - # Deprecated setup.py install will try to install under ${CMAKE_INSTALL_PREFIX}/lib64 - # so set CMAKE_INSTALL_PREFIX=/usr/ to make testing easy - - name: Build with cmake - run: | - . /opt/rh/devtoolset-7/enable - cd .. - cmake3 \ - -DCMAKE_INSTALL_PREFIX=/usr/ \ - -DPython_EXECUTABLE=$(command -v python2) \ - -DENABLE_TESTS=ON \ - -DPIP_OPTIONS="--verbose" \ - -DXRD_PYTHON_REQ_VERSION="2.7" \ - -S xrootd \ - -B build - cmake3 build -LH - cmake3 \ - --build build \ - --clean-first \ - --parallel $(($(nproc) - 1)) - cmake3 --build build --target install - python2 -m pip list - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python2 --version - python2 -m pip list - python2 -m pip show xrootd - python2 -c 'import XRootD; print(XRootD)' - python2 -c 'import pyxrootd; print(pyxrootd)' - python2 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - cmake-ubuntu-updated-python: - - # Use of sudo as https://github.com/actions/virtual-environments requires it - runs-on: ubuntu-latest - - steps: - - name: Install external dependencies with apt-get - run: | - sudo apt-get update -y - DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \ - g++ \ - git \ - cmake \ - uuid-dev \ - dpkg-dev \ - libcppunit-dev \ - libgtest-dev \ - libssl-dev \ - libx11-dev \ - python3 \ - python3-pip \ - python3-venv \ - python3-dev - sudo apt-get autoclean -y - python3 -m pip --no-cache-dir install --upgrade pip setuptools wheel - - - name: Clone repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Build with cmake - run: | - cd .. - cmake \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DPython_EXECUTABLE=$(command -v python3) \ - -DENABLE_TESTS=ON \ - -DPIP_OPTIONS="--verbose" \ - -S xrootd \ - -B build - cmake build -LH - cmake \ - --build build \ - --clean-first \ - --parallel $(($(nproc) - 1)) - sudo cmake --build build --target install - python3 -m pip list - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - cmake-macos: - - runs-on: macos-latest - - steps: - - name: Install external dependencies with homebrew - run: | - brew install \ - cmake \ - cppunit \ - make \ - gcc \ - googletest \ - zlib \ - krb5 \ - ossp-uuid \ - libxml2 \ - openssl@3 - - - name: Install necessary Python libraries - run: | - python3 -m pip install --upgrade pip setuptools wheel - python3 -m pip list - - - name: Clone repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - # Given how homebrew installs things, openssl needs to be have its locations - # be given explicitly. - - name: Build with cmake - run: | - # workaround for issue #1772, should be removed when that's fixed - sudo sed -i -e "s/localhost/localhost $(hostname)/g" /etc/hosts - cd .. - cmake \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_INSTALL_PREFIX=/usr/local/ \ - -DCMAKE_PREFIX_PATH=/usr/local/opt/openssl@3 \ - -DPython_EXECUTABLE=$(command -v python3) \ - -DENABLE_TESTS=ON \ - -DPIP_OPTIONS="--verbose" \ - -S xrootd \ - -B build - cmake build -LH - cmake \ - --build build \ - --clean-first \ - --parallel $(($(sysctl -n hw.ncpu) - 1)) - cmake --build build --target install - python3 -m pip install --user build/bindings/python - python3 -m pip list - - - name: Run tests with CTest - run: | - ctest --output-on-failure --test-dir ../build - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - export DYLD_LIBRARY_PATH=/usr/local/lib - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - rpm-centos7: - - runs-on: ubuntu-latest - container: centos:7 - - steps: - - name: Overwrite /etc/yum.repos.d/epel.repo to remove epel-source - run: | - yum install -y epel-release centos-release-scl - head -n -8 /etc/yum.repos.d/epel.repo > /tmp/epel.repo - mv -f /tmp/epel.repo /etc/yum.repos.d/epel.repo - yum clean all - - - name: Install external dependencies with yum - run: | - yum update -y - yum install --nogpg -y \ - gcc-c++ \ - rpm-build \ - git \ - python-srpm-macros \ - centos-release-scl - yum clean all - - # Need to use v1 of action as image Git is too old - - name: Clone repository now that Git is available - uses: actions/checkout@v1 - - - name: Build - run: | - cd packaging/ - ./makesrpm.sh \ - --define "_with_python3 1" \ - --define "_with_tests 1" \ - --define "_with_xrdclhttp 1" \ - --define "_with_scitokens 1" \ - --define "_with_isal 1" - yum-builddep --nogpgcheck -y *.src.rpm - mkdir RPMS - rpmbuild --rebuild \ - --define "_rpmdir RPMS/" \ - --define "_with_python3 1" \ - --define "_with_tests 1" \ - --define "_with_xrdclhttp 1" \ - --define "_with_scitokens 1" \ - --define "_with_isal 1" \ - --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" \ - *.src.rpm - - - name: Install - run: | - ls -lh packaging/RPMS/ - yum install -y \ - ./packaging/RPMS/xrootd-*.rpm \ - ./packaging/RPMS/python3-xrootd-*.rpm - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - - name: Build sdist using publishing workflow - run: | - cp packaging/wheel/* . - ./publish.sh - ls -lhtra dist/ - - rpm-fedora: - - runs-on: ubuntu-latest - container: fedora:37 - - steps: - - name: Install external dependencies with dnf - run: | - dnf update -y - dnf install --nogpg -y \ - gcc-c++ \ - rpm-build \ - tar \ - dnf-plugins-core \ - git - dnf clean all - - - name: Clone repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Build - run: | - # c.f. https://github.com/actions/checkout/issues/760 - git config --global --add safe.directory "$GITHUB_WORKSPACE" - cd packaging/ - ./makesrpm.sh \ - --define "_with_python3 1" \ - --define "_with_ceph11 1" - dnf builddep --nogpgcheck -y *.src.rpm - mkdir RPMS - rpmbuild --rebuild \ - --define "_rpmdir RPMS/" \ - --define "_with_python3 1" \ - --define "_with_ceph11 1" \ - --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" \ - *.src.rpm - - - name: Install - run: | - ls -lh packaging/RPMS/ - dnf install -y \ - ./packaging/RPMS/xrootd-*.rpm \ - ./packaging/RPMS/python3-xrootd-*.rpm - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - dpkg-ubuntu: - - # Use of sudo as https://github.com/actions/virtual-environments requires it - runs-on: ubuntu-latest - - steps: - - name: Install external dependencies with apt-get - run: | - sudo apt-get update -y - DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \ - g++ \ - git \ - cmake \ - debhelper \ - devscripts \ - equivs \ - gdebi-core - sudo apt-get autoclean -y - - - name: Clone repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Build .deb - run: | - mv packaging/debian/python3-xrootd.install.new packaging/debian/python3-xrootd.install - cp -R packaging/debian/ . - mk-build-deps --build-dep debian/control - sudo gdebi -n xrootd-build-deps-depends*.deb - version=`./genversion.sh --print-only` - dch --create -v `echo $version | sed 's/^v\(.*\)/\1/'` --package xrootd --urgency low --distribution $(lsb_release -cs) -M "This package is built and released automatically. For important notices and releases subscribe to our maling lists or visit our website." - dpkg-buildpackage -b -us -uc -tc --buildinfo-option="-udeb_packages" --buildinfo-file="deb_packages/xrootd_$(dpkg-parsechangelog -S version)_$(dpkg-architecture -qDEB_BUILD_ARCH).buildinfo" --changes-option="-udeb_packages" --buildinfo-file="deb_packages/xrootd_$(dpkg-parsechangelog -S version)_$(dpkg-architecture -qDEB_BUILD_ARCH).changes" - - - name: Install - run: | - ls -lh deb_packages/*.deb - sudo apt-get install -y \ - ./deb_packages/libxr*_*.deb \ - ./deb_packages/xrootd-libs_*.deb \ - ./deb_packages/xrootd-client*_*.deb \ - ./deb_packages/xrootd-devel_*.deb \ - ./deb_packages/xrootd-plugins_*.deb \ - ./deb_packages/xrootd-server*_*.deb \ - ./deb_packages/python3-xrootd_*.deb - - - name: Verify install - run: | - command -v xrootd - command -v xrdcp - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - sdist-centos7: - - runs-on: ubuntu-latest - container: centos:7 - - steps: - - name: Install external dependencies with yum - run: | - yum update -y - yum install -y epel-release centos-release-scl - yum clean all - yum install --nogpg -y \ - cmake3 \ - gcc-c++ \ - make \ - krb5-devel \ - libuuid-devel \ - libxml2-devel \ - openssl-devel \ - systemd-devel \ - zlib-devel \ - devtoolset-7-gcc-c++ \ - python3-devel \ - python3-setuptools \ - git \ - tree \ - cppunit-devel \ - gtest-devel - yum clean all - python3 -m pip --no-cache-dir install wheel - - # Need to use v1 of action as image Git is too old - - name: Clone repository now that Git is available - uses: actions/checkout@v1 - - - name: Build sdist using publishing workflow - run: | - cp packaging/wheel/* . - ./publish.sh - python3 -m pip --verbose install --upgrade ./dist/xrootd-*.tar.gz - python3 -m pip list - - - name: Show site-packages layout for XRootD modules - run: | - find $(python3 -c 'import XRootD; import pathlib; print(str(pathlib.Path(XRootD.__path__[0]).parent))') \ - -type d \ - -iname "*xrootd*" | xargs tree - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - sdist-centos7-updated-python: - - runs-on: ubuntu-latest - container: centos:7 - - steps: - - name: Install external dependencies with yum - run: | - yum update -y - yum install -y epel-release centos-release-scl - yum clean all - yum install --nogpg -y \ - cmake3 \ - gcc-c++ \ - make \ - krb5-devel \ - libuuid-devel \ - libxml2-devel \ - openssl-devel \ - systemd-devel \ - zlib-devel \ - devtoolset-7-gcc-c++ \ - python3-devel \ - python3-setuptools \ - git \ - tree \ - cppunit-devel \ - gtest-devel - yum clean all - python3 -m pip --no-cache-dir install --upgrade pip setuptools wheel - - # Need to use v1 of action as image Git is too old - - name: Clone repository now that Git is available - uses: actions/checkout@v1 - - - name: Build sdist using publishing workflow - run: | - cp packaging/wheel/* . - ./publish.sh - python3 -m pip --verbose install --upgrade ./dist/xrootd-*.tar.gz - python3 -m pip list - - - name: Show site-packages layout for XRootD modules - run: | - find $(python3 -c 'import XRootD; import pathlib; print(str(pathlib.Path(XRootD.__path__[0]).parent))') \ - -type d \ - -iname "*xrootd*" | xargs tree - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' - - sdist-ubuntu: - - # Use of sudo as https://github.com/actions/virtual-environments requires it - runs-on: ubuntu-latest - - steps: - - name: Install external dependencies with apt-get - run: | - sudo apt-get update -y - DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \ - g++ \ - git \ - cmake \ - uuid-dev \ - dpkg-dev \ - libssl-dev \ - libx11-dev \ - python3 \ - python3-pip \ - python3-venv \ - python3-dev \ - pkg-config \ - tree - sudo apt-get autoclean -y - # Remove packages with invalid versions which cause sdist build to fail - sudo apt-get remove python3-debian python3-distro-info - python3 -m pip --no-cache-dir install --upgrade pip setuptools wheel - python3 -m pip list - - - name: Clone repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Build sdist using publishing workflow - run: | - cp packaging/wheel/* . - ./publish.sh - python3 -m pip --verbose install --upgrade ./dist/xrootd-*.tar.gz - python3 -m pip list - - - name: Show site-packages layout for XRootD modules - run: | - find $(python3 -c 'import XRootD; import pathlib; print(str(pathlib.Path(XRootD.__path__[0]).parent))') \ - -type d \ - -iname "*xrootd*" | xargs tree - - - name: Verify Python bindings - run: | - python3 --version --version - python3 -m pip list - python3 -m pip show xrootd - python3 -c 'import XRootD; print(XRootD)' - python3 -c 'import pyxrootd; print(pyxrootd)' - python3 -c 'from XRootD import client; print(client.FileSystem("root://someserver:1094"))' diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 00000000000..b27d40c20ba --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,75 @@ +name: Python + +on: + push: + branches: + - devel + - master + paths: + - setup.py + - pyproject.toml + - bindings/python + tags-ignore: + pull_request: + paths: + - setup.py + - pyproject.toml + - bindings/python + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + manylinux: + name: Python + runs-on: ubuntu-latest + container: quay.io/pypa/manylinux_2_28_x86_64 + + strategy: + matrix: + version: [ "3.6", "3.8", "3.10", "3.11", "3.12" ] + + env: + PYTHON: python${{ matrix.version }} + + steps: + - name: Install dependencies + run: dnf install -y git krb5-devel libuuid-devel openssl-devel + + - name: Clone repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build source distribution tarball + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + ./genversion.sh >| VERSION + ${PYTHON} -m build --sdist + + - name: Build binary wheel + run: ${PYTHON} -m pip wheel --use-pep517 --verbose dist/*.tar.gz + + - name: Install binary wheel + run: ${PYTHON} -m pip install xrootd*.whl + + - name: Run post-installation tests + run: | + command -v ${PYTHON} && ${PYTHON} --version + ${PYTHON} -m pip show xrootd + ${PYTHON} -c 'import XRootD; print(XRootD)' + ${PYTHON} -c 'import pyxrootd; print(pyxrootd)' + ${PYTHON} -c 'from XRootD import client; help(client)' + ${PYTHON} -c 'from XRootD import client; print(client.FileSystem("root://localhost"))' + + - name: Move binary wheels to artifact directory + run: mv *.whl dist/ + + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: Python + path: dist + retention-days: 1 diff --git a/.gitignore b/.gitignore index 15dff770410..11165537c4d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,60 +1,7 @@ -*.o -*.lo -.libs -.deps -Makefile -Makefile.in -*.la -GNUmakefile.classic -aclocal.m4 -autom4te.cache/ -compile -config/GNUmake.rules.sunCC -config/GNUmake.rules.sunCCamd -config/GNUmake.rules.sunCCamd510 -config/GNUmake.rules.sunCCamd64 -config/GNUmake.rules.sunCCi86pc -config.guess -config.log -config.status -config.sub -configure -depcomp -install-sh -lib/ -libtool -ltmain.sh -missing -src/GNUmake.env -src/GNUmake.options -src/Makefile_include -src/XrdAcc/XrdAccTest -src/XrdApps/mpxstats -src/XrdApps/wait41 -src/XrdApps/xrdadler32 -src/XrdClient/TestXrdClient -src/XrdClient/TestXrdClient_read -src/XrdClient/XrdClientAdmin_c_wrap.cc -src/XrdClient/xprep -src/XrdClient/xrd -src/XrdClient/xrdcp -src/XrdClient/xrdstagetool -src/XrdCms/cmsd -src/XrdCns/XrdCnsd -src/XrdCns/cns_ssi -src/XrdFrm/frm_admin -src/XrdFrm/frm_purged -src/XrdFrm/frm_xfragent -src/XrdFrm/frm_xfrd -src/XrdSec/testclient -src/XrdSec/testserver -src/XrdSecgsi/xrdgsiproxy -src/XrdSecpwd/xrdpwdadmin -src/XrdSecssl/xrdsecssltest -src/XrdSecsss/xrdsssadmin -src/XrdXrootd/xrootd -test/testconfig.sh -xrootd.spec -dist +# CMake build directory +build/ +# docker builds +docker/xrootd.tar.gz +# Python build artifacts +dist/ *.egg-info -bindings/python/VERSION diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 25244c47127..5073d3c9cfe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,819 +1,97 @@ stages: - - build:rpm - - build:dockerimage:prepare - - build:dockerimage - - test - - publish - - post:publish - - clean + - build -.template:deb_ubuntu_build: &deb_ubuntu_build_def - stage: build:rpm - script: - - export DEBIAN_FRONTEND=noninteractive - - apt-get update - - apt-get install -y git cmake g++ debhelper devscripts equivs gdebi-core - - cp -R packaging/debian/ . - - mk-build-deps --build-dep debian/control - - gdebi -n xrootd-build-deps-depends*.deb - - version=`./genversion.sh --print-only` - - dch --create -v `echo $version | sed 's/^v\(.*\)/\1/'` --package xrootd --urgency low --distribution ${DIST} -M "This package is built and released automatically. For important notices and releases subscribe to our maling lists or visit our website." - - dpkg_version=`dpkg-query --showformat='${Version}' --show dpkg` - - rc=0 ; dpkg --compare-versions $dpkg_version "ge" "1.18.11" || rc=$? - - if [ $rc -eq "0" ]; then - dpkg-buildpackage -b -us -uc -tc --buildinfo-option="-udeb_packages" --changes-option="-udeb_packages" ; - else - dpkg-buildpackage -b -us -uc -tc --changes-option="-udeb_packages" ; - fi - - mkdir ${DIST}/ - - cp deb_packages/*.deb ${DIST}/ - - if [[ $DEBUG = "true" ]] ; then cp deb_packages/*.ddeb ${DIST}/; fi - artifacts: - expire_in: 1 day - paths: - - ${DIST}/ - tags: - - docker_node - -.template:deb_ubuntu_build: &deb_ubuntu_build_new_def - stage: build:rpm - script: - - export DEBIAN_FRONTEND=noninteractive - - apt-get update - - apt-get install -y git cmake g++ debhelper devscripts equivs gdebi-core - - mv packaging/debian/python3-xrootd.install.new packaging/debian/python3-xrootd.install - - cp -R packaging/debian/ . - - mk-build-deps --build-dep debian/control - - gdebi -n xrootd-build-deps-depends*.deb - - version=`./genversion.sh --print-only` - - dch --create -v `echo $version | sed 's/^v\(.*\)/\1/'` --package xrootd --urgency low --distribution ${DIST} -M "This package is built and released automatically. For important notices and releases subscribe to our maling lists or visit our website." - - dpkg-buildpackage -b -us -uc -tc --buildinfo-option="-udeb_packages" --buildinfo-file="deb_packages/xrootd_$(dpkg-parsechangelog -S version)_$(dpkg-architecture -qDEB_BUILD_ARCH).buildinfo" --changes-option="-udeb_packages" --buildinfo-file="deb_packages/xrootd_$(dpkg-parsechangelog -S version)_$(dpkg-architecture -qDEB_BUILD_ARCH).changes" - - mkdir ${DIST}/ - - cp deb_packages/*.deb ${DIST}/ - - if [[ $DEBUG = "true" ]] ; then cp deb_packages/*.ddeb ${DIST}/; fi - artifacts: - expire_in: 1 day - paths: - - ${DIST}/ - tags: - - docker_node - -build:cs9: - stage: build:rpm - image: gitlab-registry.cern.ch/linuxsupport/cs9-base - script: - - dnf install -y epel-release - - dnf install --nogpg -y gcc-c++ rpm-build tar dnf-plugins-core git - - dnf install -y cppunit-devel gtest-devel - - cd packaging/ - - ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" -D "dist .el9" - - dnf builddep --nogpgcheck -y *.src.rpm - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" -D "dist .el9" *.src.rpm - - cd .. - - mkdir cs-9 - - cp packaging/RPMS/*.rpm cs-9 - - cp packaging/*src.rpm cs-9 - artifacts: - expire_in: 1 day - paths: - - cs-9/ - tags: - - docker_node - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - allow_failure: true - -build:cs8: - stage: build:rpm - image: gitlab-registry.cern.ch/linuxsupport/cs8-base - script: - - dnf install -y epel-release - - dnf install --nogpg -y gcc-c++ rpm-build tar dnf-plugins-core git - - dnf config-manager --set-enabled powertools - - cd packaging - - ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" - - dnf builddep --nogpgcheck -y *.src.rpm - - dnf -y update libarchive - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_tests 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm - - cd .. - - mkdir cs-8 - - cp packaging/*.src.rpm cs-8 - - cp packaging/RPMS/* cs-8 - artifacts: - expire_in: 1 day - paths: - - cs-8/ +default: tags: - docker_node - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web -build:cc7: - stage: build:rpm - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - head -n -6 /etc/yum.repos.d/epel.repo > /tmp/epel.repo ; mv -f /tmp/epel.repo /etc/yum.repos.d/epel.repo - - yum install --nogpg -y gcc-c++ rpm-build git python-srpm-macros centos-release-scl - - cd packaging/ - - ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" - - yum-builddep --nogpgcheck -y *.src.rpm - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_tests 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm - - cd .. - - mkdir cc-7/ - - cp packaging/*.src.rpm cc-7 - - cp packaging/RPMS/* cc-7 - artifacts: - expire_in: 1 day - paths: - - cc-7/ - tags: - - docker_node - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - -#build:fedora-36: -# stage: build:rpm -# image: fedora:36 -# script: -# - dnf install --nogpg -y gcc-c++ rpm-build tar dnf-plugins-core git -# - cd packaging/ -# - ./makesrpm.sh --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_ceph11 1" -# - dnf builddep --nogpgcheck -y *.src.rpm -# - mkdir RPMS -# - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_ceph11 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm -# - cd .. -# - mkdir fc-rawhide -# - cp packaging/*.src.rpm fc-rawhide -# - cp packaging/RPMS/* fc-rawhide -# artifacts: -# expire_in: 1 day -# paths: -# - fc-rawhide/ -# tags: -# - docker_node -# only: -# - master -# - /^stable-.*$/ -# except: -# - tags -# - schedules -# - web - -build:fedora-35: - stage: build:rpm - image: fedora:35 - script: - - dnf install --nogpg -y gcc-c++ rpm-build tar dnf-plugins-core git - - cd packaging/ - - ./makesrpm.sh --define "_with_python3 1" --define "_with_ceph11 1" - - dnf builddep --nogpgcheck -y *.src.rpm - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_ceph11 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm - - cd .. - - mkdir fc-35/ - - cp packaging/*.src.rpm fc-35 - - cp packaging/RPMS/* fc-35 - artifacts: - expire_in: 1 day - paths: - - fc-35/ - tags: - - docker_node - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - -build:fedora-34: - stage: build:rpm - image: fedora:34 - script: - - dnf install --nogpg -y gcc-c++ rpm-build tar dnf-plugins-core git - - cd packaging/ - - ./makesrpm.sh --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_ceph11 1" - - dnf builddep --nogpgcheck -y *.src.rpm - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_ceph11 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm - - cd .. - - mkdir fc-34/ - - cp packaging/*.src.rpm fc-34 - - cp packaging/RPMS/* fc-34 - artifacts: - expire_in: 1 day - paths: - - fc-34/ - tags: - - docker_node - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - -build:fedora-36: - stage: build:rpm - image: fedora:36 - script: - - dnf install --nogpg -y gcc-c++ rpm-build tar dnf-plugins-core git - - cd packaging/ - - ./makesrpm.sh --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_ceph11 1" - - dnf builddep --nogpgcheck -y *.src.rpm - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_ceph11 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm - - cd .. - - mkdir fc-36/ - - cp packaging/*.src.rpm fc-36 - - cp packaging/RPMS/* fc-36 - artifacts: - expire_in: 1 day - paths: - - fc-36/ - tags: - - docker_node - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - allow_failure: true - -build:deb_ubuntu_focal: - image: ubuntu:focal - <<: *deb_ubuntu_build_def +.deb_build: &deb_build + stage: build variables: - DIST: focal - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - -build:deb_ubuntu_jammy: - image: ubuntu:jammy - <<: *deb_ubuntu_build_new_def - variables: - DIST: jammy - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - allow_failure: true - -build:macosx: - stage: build:rpm - script: - - mkdir build - - mkdir -p tarball/xrootd - - cd build - - cmake -DCMAKE_PREFIX_PATH='/usr/local/opt/zlib;/usr/local/opt/openssl' -DCMAKE_INSTALL_PREFIX=../tarball/xrootd .. - - cd src/XrdCl/ - - make -j4 - - make install - - cd ../../../tarball - - tar -zcf xrootd.tar.gz xrootd - - cd .. - - mkdir osx - - cp tarball/xrootd.tar.gz osx - artifacts: - expire_in: 1 day - paths: - - osx/ - tags: - - macosx-shell - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - allow_failure: true - -release:cs8-x86_64: - stage: build:rpm - image: gitlab-registry.cern.ch/linuxsupport/cs8-base - script: + DEBIAN_FRONTEND: noninteractive + script: + - source /etc/os-release + - apt update -qq + - apt install -y build-essential devscripts equivs git + - mk-build-deps --install --remove debian/control <<< y + - VERSION=$(git describe --match 'v*' | sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./') + - dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' + - debuild --no-tgz-check --no-sign -b + - apt install -y ../*.d*eb + - mkdir -p DEB/${ID}/${VERSION_CODENAME} + - mv ../*.* DEB/${ID}/${VERSION_CODENAME} + - tests/post-install.sh + artifacts: + paths: [ DEB ] + expire_in: 1d + +.rpm_build_yum: &rpm_build_yum + stage: build + script: + - yum install -y centos-release-scl epel-release git + - yum install -y epel-rpm-macros rpmdevtools yum-utils + - yum-builddep -y xrootd.spec + - rpmdev-setuptree + - git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD + - rpmbuild -bb --with git xrootd.spec + - yum install -y $(rpm -E '%{_rpmdir}')/*/*.rpm + - tests/post-install.sh + - mkdir -p RPMS + - mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) + artifacts: + paths: [ RPMS ] + expire_in: 1d + +.rpm_build_dnf: &rpm_build_dnf + stage: build + script: + - dnf install -y dnf-plugins-core git rpmdevtools + - rpmdev-setuptree + - dnf builddep -y xrootd.spec + - git archive --prefix xrootd/ -o $(rpm -E '%{_sourcedir}')/xrootd.tar.gz HEAD + - rpmbuild -bb --with git xrootd.spec + - dnf install -y $(rpm -E '%{_rpmdir}')/*/*.rpm + - tests/post-install.sh + - mkdir -p RPMS + - mv $(rpm -E '%{_rpmdir}')/ RPMS$(rpm -E '%{dist}' | tr . /) + artifacts: + paths: [ RPMS ] + expire_in: 1d + +Debian 11: + image: debian:11 + <<: *deb_build + +Debian 12: + image: debian:12 + <<: *deb_build + +Ubuntu 22.04: + image: ubuntu:22.04 + <<: *deb_build + +CentOS 7: + image: centos:7 + <<: *rpm_build_yum + +AlmaLinux 8: + image: almalinux:8 + before_script: - dnf install -y epel-release - - dnf install --nogpg -y gcc-c++ rpm-build tar dnf-plugins-core git python-macros - dnf config-manager --set-enabled powertools - - dnf install -y cppunit-devel gtest-devel - - dnf -y update libarchive - - mkdir cs-8-x86_64 - - ./gen-tarball.sh $CI_COMMIT_TAG - - mv xrootd-${CI_COMMIT_TAG#"v"}.tar.gz cs-8-x86_64 - - cd packaging/ - - git checkout tags/${CI_COMMIT_TAG} - - ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" -D "dist .el8" - - dnf builddep --nogpgcheck -y *.src.rpm - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" -D "dist .el8" *.src.rpm - - cd .. - - cp packaging/RPMS/*.rpm cs-8-x86_64 - - cp packaging/*src.rpm cs-8-x86_64 - artifacts: - expire_in: 1 day - paths: - - cs-8-x86_64/ - tags: - - docker_node - only: - - web - except: - - branches - -release:rocky8-x86_64: - stage: build:rpm - image: rockylinux:8 - script: - - dnf install -y epel-release - - dnf install --nogpg -y gcc-c++ rpm-build tar dnf-plugins-core git python-macros - - dnf config-manager --set-enabled powertools - - dnf install -y cppunit-devel gtest-devel - - dnf -y update libarchive - - mkdir rocky-8-x86_64 - - ./gen-tarball.sh $CI_COMMIT_TAG - - mv xrootd-${CI_COMMIT_TAG#"v"}.tar.gz cs-8-x86_64 - - cd packaging/ - - git checkout tags/${CI_COMMIT_TAG} - - ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" -D "dist .el8" - - dnf builddep --nogpgcheck -y *.src.rpm - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" -D "dist .el8" *.src.rpm - - cd .. - - cp packaging/RPMS/*.rpm rocky-8-x86_64 - - cp packaging/*src.rpm rocky-8-x86_64 - artifacts: - expire_in: 1 day - paths: - - rocky-8-x86_64/ - tags: - - docker_node - only: - - web - except: - - branches - -release:cc7-x86_64: - stage: build:rpm - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - head -n -6 /etc/yum.repos.d/epel.repo > /tmp/epel.repo ; mv -f /tmp/epel.repo /etc/yum.repos.d/epel.repo - - yum install --nogpg -y gcc-c++ rpm-build git python-srpm-macros centos-release-scl - - mkdir cc-7-x86_64 - - ./gen-tarball.sh $CI_COMMIT_TAG - - mv xrootd-${CI_COMMIT_TAG#"v"}.tar.gz cc-7-x86_64 - - cd packaging/ - - git checkout tags/${CI_COMMIT_TAG} - - ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" -D "dist .el7" - - yum-builddep --nogpgcheck -y *.src.rpm - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_python3 1" --define "_with_tests 1" --define "_without_xrootd_user 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" -D "dist .el7" *.src.rpm - - cd .. - - cp packaging/RPMS/*.rpm cc-7-x86_64 - - cp packaging/*src.rpm cc-7-x86_64 - artifacts: - expire_in: 1 day - paths: - - cc-7-x86_64/ - tags: - - docker_node - only: - - web - except: - - branches - -release:deb_ubuntu_focal: - image: ubuntu:focal - <<: *deb_ubuntu_build_def - variables: - DIST: focal - only: - - web - except: - - branches - -release:deb_ubuntu_jammy: - image: ubuntu:jammy - <<: *deb_ubuntu_build_new_def - variables: - DIST: jammy - only: - - web - except: - - branches - allow_failure: true - + <<: *rpm_build_dnf -release:pypi: - stage: build:rpm - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - yum install --nogpg -y git python3-pip - - cp packaging/wheel/* . - - ./publish.sh - artifacts: - expire_in: 1 day - paths: - - dist/ - tags: - - docker_node - only: - - web - except: - - branches - allow_failure: true - -publish:pypi: - stage: publish - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - yum install --nogpg -y sssd-client sudo - - sudo -u stci -H mkdir -p /eos/project/s/storage-ci/www/xrootd/release/pypi-dist - - sudo -u stci -H cp dist/*.tar.gz /eos/project/s/storage-ci/www/xrootd/release/pypi-dist/. - tags: - - docker_node - only: - - web - except: - - branches - allow_failure: true - - -weekly:cs8: - stage: build:rpm - image: gitlab-registry.cern.ch/linuxsupport/cs8-base - script: +AlmaLinux 9: + image: almalinux:9 + before_script: - dnf install -y epel-release - - dnf install --nogpg -y gcc-c++ rpm-build tar dnf-plugins-core git python-macros - - dnf config-manager --set-enabled powertools - - dnf install -y cppunit-devel gtest-devel - - dnf -y update libarchive - - xrootd_version=$(git for-each-ref --sort=taggerdate --format '%(refname)' refs/tags | grep '^refs/tags/v' | grep -v 'rc.*$' | grep -v 'osghotfix' | grep -v 'CERN$' | sed -e '$!d') - - xrootd_version=${xrootd_version:11} - - short_hash=$(git rev-parse --verify HEAD | awk '{print substr($0, 0, 8)}') - - a=( ${xrootd_version//./ } ) - - ((a[2]++)) || true - - experimental_version="${a[0]}.${a[1]}.${a[2]}-0.experimental."${CI_PIPELINE_ID}.$short_hash - - cd packaging/ - - ./makesrpm.sh --version $experimental_version --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" - - dnf builddep --nogpgcheck -y *.src.rpm - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_tests 1" --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm - - cd .. - - mkdir epel-8/ - - cp packaging/*.src.rpm epel-8 - - cp packaging/RPMS/* epel-8 - artifacts: - expire_in: 1 day - paths: - - epel-8/ - tags: - - docker_node - only: - - schedules - except: - - tags - -weekly:cc7: - stage: build:rpm - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - head -n -6 /etc/yum.repos.d/epel.repo > /tmp/epel.repo ; mv -f /tmp/epel.repo /etc/yum.repos.d/epel.repo - - yum install --nogpg -y gcc-c++ rpm-build git cppunit-devel gtest-devel python-srpm-macros centos-release-scl - - xrootd_version=$(git for-each-ref --sort=taggerdate --format '%(refname)' refs/tags | grep '^refs/tags/v5' | grep -v 'rc.*$' | grep -v 'osghotfix' | grep -v 'CERN$' | sed -e '$!d') - - xrootd_version=${xrootd_version:11} - - echo $xrootd_version - - short_hash=$(git rev-parse --verify HEAD | awk '{print substr($0, 0, 8)}') - - a=( ${xrootd_version//./ } ) - - ((a[2]++)) || true - - echo $CI_PIPELINE_ID - - experimental_version="${a[0]}.${a[1]}.${a[2]}-0.experimental."${CI_PIPELINE_ID}.$short_hash - - echo $experimental_version - - cd packaging/ - - ./makesrpm.sh --version $experimental_version --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" - - yum-builddep --nogpgcheck -y *.src.rpm - - mkdir RPMS - - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_with_tests 1" --define "_with_python3 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm - - cd .. - - mkdir epel-7/ - - cp packaging/*.src.rpm epel-7 - - cp packaging/RPMS/* epel-7 - artifacts: - expire_in: 1 day - paths: - - epel-7/ - tags: - - docker_node - only: - - schedules - - web - except: - - tags + - dnf config-manager --set-enabled crb + <<: *rpm_build_dnf -xrootd_docker_get: - stage: build:dockerimage:prepare - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - yum install --nogpg -y git - - git clone https://gitlab.cern.ch/eos/xrootd-docker.git - - if [ ! -d "epel-7" ]; then mkdir epel-7; cp cc-7/* epel-7; fi - artifacts: - expire_in: 1 day - paths: - - xrootd-docker/ - - epel-7/ - tags: - - docker_node - only: - - web - - schedules - except: - - tags - -xrootd_dockerimage: - stage: build:dockerimage - tags: - - docker-image-build - script: - - "" - variables: - TO: gitlab-registry.cern.ch/dss/xrootd - DOCKER_FILE: xrootd-docker/Dockerfile.ci - dependencies: - - xrootd_docker_get - only: - - schedules - - web - except: - - tags - -xrootd_docker_test: - stage: test - script: - - docker pull gitlab-registry.cern.ch/dss/xrootd - - cd xrootd-docker - - yum -y install wget - - sudo ./start.sh -i gitlab-registry.cern.ch/dss/xrootd - - docker exec metaman text-runner /usr/lib64/libXrdClTests.so "All Tests/UtilsTest/" - - docker exec metaman text-runner /usr/lib64/libXrdClTests.so "All Tests/SocketTest/" - - docker exec metaman text-runner /usr/lib64/libXrdClTests.so "All Tests/PollerTest/" - - docker exec metaman text-runner /usr/lib64/libXrdClTests.so "All Tests/PostMasterTest/" - - docker exec metaman text-runner /usr/lib64/libXrdClTests.so "All Tests/FileSystemTest/" -## - docker exec metaman text-runner /usr/lib64/libXrdClTests.so "All Tests/FileTest/" - - docker exec metaman text-runner /usr/lib64/libXrdClTests.so "All Tests/LocalFileHandlerTest/" - - after_script: - - sudo ./xrootd-docker/clean.sh - tags: - - shell-with-docker - dependencies: - - xrootd_docker_get - only: - - schedules - - web - except: - - tags - allow_failure: true - -pyxrootd_dockerimage: - stage: build:dockerimage - tags: - - docker-image-build - script: - - "" - variables: - TO: gitlab-registry.cern.ch/dss/xrootd:pylatest - DOCKER_FILE: xrootd-docker/Dockerfile-python.ci - dependencies: - - xrootd_docker_get - only: - - schedules - - web - except: - - tags - allow_failure: true - -pyxrootd_docker_test: - stage: test - script: - - docker pull gitlab-registry.cern.ch/dss/xrootd:pylatest - - sudo docker run -dit --privileged -e "container=docker" --name pyxrootd-container -h pyxrootd-container gitlab-registry.cern.ch/dss/xrootd:pylatest /sbin/init - - docker exec pyxrootd-container systemctl start xrootd@standalone - - docker exec pyxrootd-container sh -c "cd xrootd/bindings/python/tests && pytest test_file.py test_filesystem.py test_copy.py test_threads.py test_url.py" - - after_script: - - sudo docker rm -f pyxrootd-container - - sudo docker rmi -f gitlab-registry.cern.ch/dss/xrootd:pylatest - - tags: - - shell-with-docker - dependencies: - - xrootd_docker_get - only: - - schedules - - web - except: - - tags - allow_failure: true - -publish:rhel: - stage: publish - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - yum install --nogpg -y sssd-client sudo createrepo - - prefix=/eos/project/s/storage-ci/www/xrootd - - "for platform in cs-8 cc-7 fc-35 fc-34; do - repo=$prefix/${CI_COMMIT_REF_NAME}/$platform/x86_64 - path=$repo/$(date +'%Y%m%d'); - sudo -u stci -H mkdir -p $path; - sudo -u stci -H find ${path} -type f -name '*.rpm' -delete; - sudo -u stci -H cp $platform/* $path; - sudo -u stci -H createrepo --update -q $path; - sudo -u stci -H createrepo --update -q $repo; - done" - tags: - - docker_node - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - -publish:debian: - stage: publish - image: ubuntu:jammy - script: - - apt-get update - - apt-get install -y sudo apt-utils sssd gpg - - mkdir /home/stci - - chown -R stci:def-cg /home/stci - - chmod -R 700 /home/stci - - sudo -u stci -H gpg --homedir /home/stci/ --allow-secret-key-import --import /keys/stci-debian-repo.sec - - sudo -u stci -H ./packaging/debian_scripts/publish_debian_cern.sh ${CI_COMMIT_REF_NAME} - tags: - - docker_node - dependencies: - - build:deb_ubuntu_focal - - build:deb_ubuntu_jammy - only: - - master - - /^stable-.*$/ - except: - - tags - - schedules - - web - -publish:rhel:release: - stage: publish - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - yum install --nogpg -y sssd-client sudo createrepo - - prefix=/eos/project/s/storage-ci/www/xrootd - - tarball=cc-7-x86_64/xrootd-*.tar.gz - - "for platform in rocky-8-x86_64 cs-8-x86_64 cc-7-x86_64; do - path=$prefix/release/$platform/$CI_COMMIT_TAG/; - sudo -u stci -H mkdir -p $path/{source,tarball}; - sudo -u stci -H cp $platform/*.rpm $path; - sudo -u stci -H find ${path} -type f -name '*.src.rpm' -delete; - sudo -u stci -H cp $platform/*.src.rpm $path/source; - sudo -u stci -H cp $tarball $path/tarball; - sudo -u stci -H createrepo --update -q $path; - sudo -u stci -H createrepo --update -q $prefix/release/$platform; - done" - tags: - - docker_node - only: - - web - except: - - branches - -publish:debian:release: - stage: publish - image: ubuntu:jammy - script: - - apt-get update - - apt-get install -y sudo apt-utils sssd gpg - - mkdir /home/stci - - chown -R stci:def-cg /home/stci - - chmod -R 700 /home/stci - - sudo -u stci -H gpg --homedir /home/stci/ --allow-secret-key-import --import /keys/stci-debian-repo.sec - - repo=release - - if [[ $CI_COMMIT_TAG == *rc* ]] ; then repo=testing ; fi - - sudo -u stci -H ./packaging/debian_scripts/publish_debian_cern.sh $repo - tags: - - docker_node - dependencies: - - release:deb_ubuntu_focal - - release:deb_ubuntu_jammy - only: - - web - except: - - branches - -publish:weekly: - stage: publish - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - yum install --nogpg -y sssd-client sudo createrepo - - prefix=/eos/project/s/storage-ci/www/xrootd - - "for platform in epel-8 epel-7 epel-6; do - if [ -d $platform ] ; then - path=$prefix/experimental/$platform/x86_64/; - sudo -u stci -H mkdir -p $path; - ls -latr $platform/; - echo $path; - sudo -u stci -H cp $platform/* $path; - sudo -u stci -H createrepo --update -q $path; - fi; - done" - tags: - - docker_node - dependencies: - - weekly:cc7 - - weekly:cs8 - only: - - schedules - - web - except: - - tags - -publish:koji:cs8: - stage: post:publish - image: gitlab-registry.cern.ch/linuxsupport/rpmci/kojicli - script: - - yum install --nogpg -y sssd-client - - kinit stci@CERN.CH -k -t /stci.krb5/stci.keytab - - path=/eos/project/s/storage-ci/www/xrootd/release/cs-8-x86_64/$CI_COMMIT_TAG/source/ - - if [[ $CI_COMMIT_TAG != *rc* ]] ; then koji build eos8 $path/*.src.rpm ; else stat $path/*.src.rpm ; fi - tags: - - docker_node - only: - - web - except: - - branches - when: manual - -publish:koji:cc7: - stage: post:publish - image: gitlab-registry.cern.ch/linuxsupport/rpmci/kojicli - script: - - yum install --nogpg -y sssd-client - - kinit stci@CERN.CH -k -t /stci.krb5/stci.keytab - - path=/eos/project/s/storage-ci/www/xrootd/release/cc-7-x86_64/$CI_COMMIT_TAG/source/ - - if [[ $CI_COMMIT_TAG != *rc* ]] ; then koji build eos7 $path/*.src.rpm ; else stat $path/*.src.rpm ; fi - tags: - - docker_node - only: - - web - except: - - branches - when: manual - -clean:artifacts: - stage: clean - image: gitlab-registry.cern.ch/linuxsupport/cc7-base - script: - - yum install --nogpg -y sssd-client sudo createrepo - - sudo -u stci -H bash -c 'for commit_dir in /eos/project/s/storage-ci/www/xrootd/master/*/*/; do find ${commit_dir} -mindepth 1 -maxdepth 1 -type d -ctime +10 | xargs rm -rf; createrepo --update -q ${commit_dir}; done' - - sudo -u stci -H bash -c 'for commit_dir in /eos/project/s/storage-ci/www/xrootd/stable-*/*/*/; do find ${commit_dir} -type f -name '"'"'*.rpm'"'"' -mtime +30 -delete; createrepo --update -q ${commit_dir}; done' - - sudo -u stci -H bash -c 'for commit_dir in /eos/project/s/storage-ci/www/xrootd/experimental/*/x86_64/; do find ${commit_dir} -type f -name '"'"'*.rpm'"'"' -mtime +30 -delete; createrepo --update -q ${commit_dir}; done' - tags: - - docker_node - allow_failure: true - only: - - schedules - - web - except: - - tags +Fedora 38: + image: fedora:38 + <<: *rpm_build_dnf +Fedora 39: + image: fedora:39 + <<: *rpm_build_dnf diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 6a3084b7891..00000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "src/XrdCeph"] - path = src/XrdCeph - url = https://github.com/xrootd/xrootd-ceph diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000000..aa20ef11ac5 --- /dev/null +++ b/.mailmap @@ -0,0 +1,79 @@ +Alja Mrak-Tadel Alja MRak-Tadel +Alja Mrak-Tadel alja +Alja Mrak-Tadel alja +Alja Mrak-Tadel alja +Andreas Joachim Peters Andreas Peters +Andreas Joachim Peters Andreas Peters +Andreas Joachim Peters Andreas-Joachim Peters +Andrew Hanushevsky +Artem Harutyunyan +Brian Bockelman Brian P Bockelman +Brian Bockelman Brian P Bockelman +Cedric Caffy ccaffy <85744538+ccaffy@users.noreply.github.com> +Chris Burr Chris Burr +Chris Green +David Smith +David Smith David Smith +Ed J +Edgar Fajardo efajardo +Edgar Fajardo efajardo +Elvin Sindrilaru +Fabrizio Furano Fabrizio Furano +Fabrizio Furano Fabrizio Furano +Fabrizio Furano Fabrizio Furano +Fabrizio Furano Fabrizio Furano +Fabrizio Furano ffurano +Fabrizio Furano furano +Frank Winklmeier fwinkl <> +Fritz Mueller Fritz Mueller +Gerardo Ganis Gerardo GANIS +Gerardo Ganis Gerri Ganis +Gerardo Ganis Gerri Ganis +Gerardo Ganis Gerri Ganis +Gerardo Ganis ganis +Gerardo Ganis ganis +Gerardo Ganis gganis +Jacek Becla +James Walder snafus +Jan Iven +Jozsef Makai +Jozsef Makai Jozsef Makai +Jyothish Thomas Jo-stfc <71326101+Jo-stfc@users.noreply.github.com> +Jyothish Thomas root +Kian-Tat Lim ktlim +Lukasz Janyst Lukasz Janyst +Lukasz Janyst Lukasz Janyst +Lukasz Janyst Lukasz Janyst +Lukasz Janyst Lukasz Janyst +Matevž Tadel Matevz Tadel +Matevž Tadel Matevž Tadel +Michal Simon Michal Simon +Michal Simon Michal Simon +Michal Simon simonmichal +Nikola Hardi +Nikola Hardi +Paul-Niklas Kramp Paul Kramp +Paul-Niklas Kramp niklas +Paul-Niklas Kramp pkramp +Remigius Mommsen mommsen <> +Sebastien Ponce +Thorsten Kollegger TKollegger +Wei Yang Wei Yang +Wei Yang Wei Yang +Wei Yang Wei Yang +Wei Yang Wei Yang +Wei Yang Wei Yang +Wei Yang Wei Yang +Wei Yang Wei Yang +Wei Yang Wei Yang +Wei Yang Wei Yang +Wei Yang Wei Yang + +Unknown bbrqa <> +Unknown otron +Unknown root +Unknown root +Unknown root +Unknown root +Unknown xrootd +Unknown root diff --git a/CMakeLists.txt b/CMakeLists.txt index ec47ab07053..c771264ff36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,8 @@ set( CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/cmake ) +include(XRootDVersion) + #------------------------------------------------------------------------------- # A 'plugins' phony target to simplify building build-tree binaries. # Plugins are responsible for adding themselves to this target, where @@ -30,57 +32,23 @@ add_definitions( -DXRDPLUGIN_SOVERSION="${PLUGIN_VERSION}" ) #------------------------------------------------------------------------------- # Generate the version header #------------------------------------------------------------------------------- -if (USER_VERSION) - set(XROOTD_VERSION "${USER_VERSION}") -else () -execute_process( - COMMAND ${CMAKE_SOURCE_DIR}/genversion.sh --print-only ${CMAKE_SOURCE_DIR} - OUTPUT_VARIABLE XROOTD_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE ) -endif() - -add_custom_target( - XrdVersion.hh - ${CMAKE_SOURCE_DIR}/genversion.sh --version ${XROOTD_VERSION} ${CMAKE_SOURCE_DIR}) - -# sigh, yet another ugly hack :( -macro( add_library _target ) - _add_library( ${_target} ${ARGN} ) - add_dependencies( ${_target} XrdVersion.hh ) -endmacro() - -macro( add_executable _target ) - _add_executable( ${_target} ${ARGN} ) - add_dependencies( ${_target} XrdVersion.hh ) -endmacro() -#------------------------------------------------------------------------------- -# Checkout the vomsxrd submodule -#------------------------------------------------------------------------------- -find_package(Git QUIET) -if(GIT_FOUND AND EXISTS "${PROJECT_SOURCE_DIR}/.git") - option(GIT_SUBMODULES "Check submodules during build" ON) - if(GIT_SUBMODULES) - message(STATUS "Submodule update") - execute_process(COMMAND ${GIT_EXECUTABLE} submodule update --init --recursive - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - RESULT_VARIABLE GIT_SUBMOD_RESULT) - if(NOT GIT_SUBMOD_RESULT EQUAL "0") - message(FATAL_ERROR "git submodule update --init failed with ${GIT_SUBMOD_RESULT}, please checkout submodules") - endif() - endif() -endif() +configure_file(src/XrdVersion.hh.in src/XrdVersion.hh) #------------------------------------------------------------------------------- # Build in subdirectories #------------------------------------------------------------------------------- -add_subdirectory( src ) -add_subdirectory( bindings ) -if( BUILD_TESTS ) - ENABLE_TESTING() - add_subdirectory( tests ) -endif() +include_directories(BEFORE ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src) + +include(CTest) + +add_subdirectory(src) +add_subdirectory(bindings) +add_subdirectory(tests) + +add_subdirectory(docs) +add_subdirectory(utils) include( XRootDSummary ) @@ -89,11 +57,24 @@ include( XRootDSummary ) # Install XRootDConfig.cmake module #------------------------------------------------------------------------------- -configure_file( "cmake/XRootDConfig.cmake.in" "cmake/XRootDConfig.cmake" @ONLY ) +include(CMakePackageConfigHelpers) + +write_basic_package_version_file(cmake/${PROJECT_NAME}ConfigVersion.cmake + VERSION ${XRootD_VERSION} COMPATIBILITY SameMajorVersion) + +configure_package_config_file(cmake/${PROJECT_NAME}Config.cmake.in cmake/${PROJECT_NAME}Config.cmake + INSTALL_PREFIX + ${CMAKE_INSTALL_PREFIX} + INSTALL_DESTINATION + ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} + PATH_VARS + CMAKE_INSTALL_INCLUDEDIR + CMAKE_INSTALL_LIBDIR + CMAKE_INSTALL_DATADIR +) -install( - FILES ${CMAKE_BINARY_DIR}/cmake/XRootDConfig.cmake - DESTINATION ${CMAKE_INSTALL_DATADIR}/xrootd/cmake ) +install(DIRECTORY ${PROJECT_BINARY_DIR}/cmake/ + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}) #------------------------------------------------------------------------------- # Configure an 'uninstall' target diff --git a/CTestConfig.cmake b/CTestConfig.cmake new file mode 100644 index 00000000000..f2ce8e263fe --- /dev/null +++ b/CTestConfig.cmake @@ -0,0 +1,4 @@ +set(CTEST_PROJECT_NAME "XRootD") +set(CTEST_NIGHTLY_START_TIME "00:00:00 UTC") +set(CTEST_DROP_SITE_CDASH TRUE) +set(CTEST_SUBMIT_URL https://my.cdash.org/submit.php?project=XRootD) diff --git a/packaging/wheel/MANIFEST.in b/MANIFEST.in similarity index 75% rename from packaging/wheel/MANIFEST.in rename to MANIFEST.in index d89d3e218a1..e348ad7b7e9 100644 --- a/packaging/wheel/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,13 @@ include *.sh *.py *.in -include CMakeLists.txt VERSION_INFO README COPYING* LICENSE +include CMakeLists.txt +include COPYING* LICENSE +include VERSION README.md recursive-include bindings * recursive-include cmake * +recursive-include docs * recursive-include packaging * recursive-include src * recursive-include tests * recursive-include ups * recursive-include utils * -recursive-include docs * diff --git a/README b/README deleted file mode 100644 index 4df5d8ff7b2..00000000000 --- a/README +++ /dev/null @@ -1,83 +0,0 @@ - --------------------------------------------------------------------------------- - _ _ ______ _____ - \ \ / (_____ \ _ (____ \ - \ \/ / _____) ) ___ ___ | |_ _ \ \ - ) ( (_____ ( / _ \ / _ \| _)| | | | - / /\ \ | | |_| | |_| | |__| |__/ / - /_/ \_\ |_|\___/ \___/ \___)_____/ - --------------------------------------------------------------------------------- - -1. S U P P O R T E D O P E R A T I N G S Y S T E M S - - XRootD is supported on the following platforms: - - * RedHat Enterprise Linux 7 or greater and their derivatives (Scientific Linux) - compiled with gcc - * MacOSX 10.6 or greater compiled with gcc or clang - -2. B U I L D I N S T R U C T I O N S - -2.0 Build dependecies - - XRootD requires at minimum following packages (RHEL distro): - - * gcc-c++ cmake(3) krb5-devel libuuid-devel libxml2-devel openssl-devel systemd-devel zlib-devel - * devtoolset-7 (only RHEL7) - -2.1 Build system - - XRootD uses CMake to handle the build process. It should build fine with -cmake 3.6 or greater. It may be also possible to use version 2.8 but this -is neither recommended nor supported. - -2.2 Build parameters - - The build process supports the following parameters: - - * CMAKE_INSTALL_PREFIX - indicates where the XRootD files should be installed, - (default: /usr) - * CMAKE_BUILD_TYPE - type of the build: Release/Debug/RelWithDebInfo - * FORCE_32BITS - Force building 32 bit binaries when on Solaris AMD64 - (default: FALSE) - * ENABLE_PERL - enable the perl bindings if possible (default: TRUE) - * ENABLE_FUSE - enable the fuse filesystem driver if possible - (default: TRUE) - * ENABLE_KRB5 - enable the Kerberos 5 authentication if possible - (default: TRUE) - * ENABLE_READLINE - enable the lib readline support in the commandline - utilities (default: TRUE) - * OPENSSL_ROOT_DIR - path to the root of the openssl installation if it - cannot be detected in a standard location - * KERBEROS5_ROOT_DIR - path to the root of the kerberos installation if it - cannot be detected in a standard location - * READLINE_ROOT_DIR - path to the root of the readline installation if it - cannot be detected in a standard location - * CMAKE_C_COMPILER - path to the c compiler that should be used - * CMAKE_CXX_COMPILER - path to the c++ compiler that should be used - -2.3 Build steps - - * on RHEL7 only: scl enable devtoolset-7 /bin/bash - - * Create an empty build directory: - - mkdir build - cd build - - * Generate the build system files using cmake, ie: - - cmake /path/to/the/xrootd/source -DCMAKE_INSTALL_PREFIX=/opt/xrootd \ - -DENABLE_PERL=FALSE - - * Build the source: - - make - - * Install the source: - - make install - -3. P L A T F O R M N O T E S - * None. diff --git a/README.md b/README.md new file mode 100644 index 00000000000..a21832d4c22 --- /dev/null +++ b/README.md @@ -0,0 +1,117 @@ +

+ +

+ +## XRootD: eXtended ROOT Daemon + +The [XRootD](http://xrootd.org) project provides a high-performance, +fault-tolerant, and secure solution for handling massive amounts of data +distributed across multiple storage resources, such as disk servers, tape +libraries, and remote sites. It enables efficient data access and movement in a +transparent and uniform manner, regardless of the underlying storage technology +or location. It was initially developed by the High Energy Physics (HEP) +community to meet the data storage and access requirements of the BaBar +experiment at SLAC and later extended to meet the needs of experiments at the +Large Hadron Collider (LHC) at CERN. XRootD is the core technology powering the +[EOS](https://eos-web.web.cern.ch/) distributed filesystem, which is the storage +solution used by LHC experiments and the storage backend for +[CERNBox](https://cernbox.web.cern.ch/). XRootD is also used as the core +technology for global CDN deployments across multiple science domains. + +XRootD is based on a scalable architecture that supports multi-protocol +communications. XRootD provides a set of plugins and tools that allows the user +to configure it freely to deploy data access clusters of any size, and which can +include sophisticated features such as erasure coded files, various methods of +authentication and authorization, as well as integration with other storage +systems like [ceph](https://ceph.io). + +## Documentation + +General documentation such as configuration reference guides, and user manuals +can be found on the XRootD website at http://xrootd.org/docs.html. + +## Supported Operating Systems + +XRootD is officially supported on the following platforms: + + * RedHat Enterprise Linux 7 or later and their derivatives + * Debian 11 and Ubuntu 22.04 or later + * macOS 11 (Big Sur) or later + +Support for other operating systems is provided on a best-effort basis +and by contributions from the community. + +## Installation Instructions + +XRootD is available via official channels in most operating systems. +Installation via your system's package manager should be preferred. + +In RPM-based distributions, like CentOS, Alma, Rocky, Fedora, etc, one can +search and install XRootD packages with + +```sh +$ sudo yum install xrootd +``` +or +```sh +$ sudo dnf install xrootd +``` + +In RHEL-based distributions, it will be necessary to first install the EPEL +release repository with `yum install epel-release` or `dnf install epel-release`. + +If you would like to use our official repository for XRootD RPMs, you can enable +it on RHEL-based distributions with + +```sh +$ sudo curl -L https://cern.ch/xrootd/xrootd.repo -o /etc/yum.repos.d/xrootd.repo +``` + +and on Fedora with +```sh +$ sudo curl -L https://cern.ch/xrootd/xrootd-fedora.repo -o /etc/yum.repos.d/xrootd.repo +``` + +On Debian 11 or later, and Ubuntu 22.04 or later, XRootD can be installed via apt + +```sh +$ sudo apt install xrootd-client xrootd-server python3-xrootd +``` + +On macOS, XRootD is available via Homebrew +```sh +$ brew install xrootd +``` + +XRootD can also be installed with conda, as it is also available in conda-forge: +```sh +$ conda config --add channels conda-forge +$ conda config --set channel_priority strict +$ conda install xrootd +``` + +Finally, it is possible to install the XRootD python bindings from PyPI using pip: +```sh +$ pip install xrootd +``` + +For detailed instructions on how to build and install XRootD from source code, +please see [docs/INSTALL.md](https://github.com/xrootd/xrootd/blob/master/docs/INSTALL.md) +in the main repository on GitHub. + +## User Support and Bug Reports + +Bugs should be reported using [GitHub issues](https://github.com/xrootd/xrootd/issues). +You can open a new ticket by clicking [here](https://github.com/xrootd/xrootd/issues/new). + +For general questions about XRootD, please send a message to our user mailing +list at xrootd-l@slac.stanford.edu or open a new [discussion](https://github.com/xrootd/xrootd/discussions) +on GitHub. Please check XRootD's contact page at http://xrootd.org/contact.html +for further information. + +## Contributing + +User contributions can be submitted via pull request on GitHub. We recommend +that you create your own fork of XRootD on GitHub and use it to submit your +patches. For more detailed instructions on how to contribute, please refer to +the file [docs/CONTRIBUTING.md](https://github.com/xrootd/xrootd/blob/master/docs/CONTRIBUTING.md). diff --git a/VERSION b/VERSION new file mode 100644 index 00000000000..f4034a5d279 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +$Format:%(describe)$ diff --git a/bindings/python/.gitignore b/bindings/python/.gitignore index 22eb52622ed..a4103cedce0 100755 --- a/bindings/python/.gitignore +++ b/bindings/python/.gitignore @@ -4,4 +4,3 @@ build .cproject *.py[co] MANIFEST -VERSION_INFO diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 2c5168475f4..c6a77f5aa0d 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -1,109 +1,33 @@ +cmake_minimum_required(VERSION 3.16...3.25) -set(SETUP_PY_IN "${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in") -set(SETUP_PY "${CMAKE_CURRENT_BINARY_DIR}/setup.py") -set(DEPS "${CMAKE_CURRENT_SOURCE_DIR}/libs/__init__.py") -set(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/python_bindings") -set(XRD_SRCINCDIR "${CMAKE_SOURCE_DIR}/src") -set(XRD_BININCDIR "${CMAKE_BINARY_DIR}/src") -set(XRDCL_LIBDIR "${CMAKE_BINARY_DIR}/src/XrdCl") -set(XRD_LIBDIR "${CMAKE_BINARY_DIR}/src") -set(XRDCL_INSTALL "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") +project(PyXRootD LANGUAGES CXX) -if( PYPI_BUILD ) - set(XRDCL_RPATH "$ORIGIN/${CMAKE_INSTALL_LIBDIR}") +if( CMAKE_VERSION VERSION_LESS 3.18 ) + set(PYTHON_COMPONENTS Interpreter Development) else() - set(XRDCL_RPATH "$ORIGIN/../../..") + set(PYTHON_COMPONENTS Interpreter Development.Module) endif() -if( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" ) - if( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.7 ) - message( "clang 3.5" ) - set( CLANG_PROHIBITED ", '-Wp,-D_FORTIFY_SOURCE=2', '-fstack-protector-strong'" ) - endif() - if( ( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 6.0 ) OR ( CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0 ) ) - message( "clang 6.0" ) - set( CLANG_PROHIBITED ", '-fcf-protection'" ) - endif() - if( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0 ) - message( "clang > 7.0" ) - set( CLANG_PROHIBITED ", '-fstack-clash-protection'" ) - endif() -endif() +find_package(Python REQUIRED COMPONENTS ${PYTHON_COMPONENTS}) -configure_file(${SETUP_PY_IN} ${SETUP_PY}) - -string(FIND "${PIP_OPTIONS}" "--prefix" PIP_OPTIONS_PREFIX_POSITION) -if( "${PIP_OPTIONS_PREFIX_POSITION}" EQUAL "-1" ) - string(APPEND PIP_OPTIONS " --prefix \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}") +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR OR PYPI_BUILD) + add_subdirectory(src) else() - message(WARNING - " The pip option --prefix has been set in '${PIP_OPTIONS}' which will change" - " it from its default value of '--prefix \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}'." - " Make sure this is intentional and that you understand the effects." - ) -endif() - -# Check it the Python interpreter has a valid version of pip -execute_process( - COMMAND ${Python_EXECUTABLE} -m pip --version - RESULT_VARIABLE VALID_PIP_EXIT_CODE - OUTPUT_QUIET -) + configure_file(setup.py setup.py) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/VERSION "${XRootD_VERSION_STRING}") -if ( NOT ${VALID_PIP_EXIT_CODE} EQUAL 0 ) - # Attempt to still install with deprecated invocation of setup.py - message(WARNING - " ${Python_EXECUTABLE} does not have a valid pip associated with it." - " It is recommended that you install a version of pip to install Python" - " packages and bindings. If you are unable to install a version of pip" - " through a package manager or with your Python build try using the PyPA's" - " get-pip.py bootstrapping script ( https://github.com/pypa/get-pip ).\n" - " The installation of the Python bindings will attempt to continue using" - " the deprecated method of `${Python_EXECUTABLE} setup.py install`." - ) + option(INSTALL_PYTHON_BINDINGS "Install Python bindings" TRUE) - # https://docs.python.org/3/install/#splitting-the-job-up - add_custom_command(OUTPUT ${OUTPUT} - COMMAND ${Python_EXECUTABLE} ${SETUP_PY} --verbose build - DEPENDS ${DEPS}) + if(INSTALL_PYTHON_BINDINGS) + set(PIP_OPTIONS "" CACHE STRING "Install options for pip") - add_custom_target(python_target ALL DEPENDS ${OUTPUT} XrdCl) - - # Get the distribution name on Debian families - execute_process( COMMAND grep -i ^NAME= /etc/os-release - OUTPUT_VARIABLE DEB_DISTRO ) - STRING(REGEX REPLACE "^NAME=\"" "" DEB_DISTRO "${DEB_DISTRO}") - STRING(REGEX REPLACE "\".*" "" DEB_DISTRO "${DEB_DISTRO}") - - if( DEB_DISTRO STREQUAL "Debian" OR DEB_DISTRO STREQUAL "Ubuntu" ) - set(PYTHON_LAYOUT "unix" CACHE STRING "Python installation layout (deb or unix)") - set(DEB_INSTALL_ARGS "--install-layout ${PYTHON_LAYOUT}") - endif() - - install( - CODE - "EXECUTE_PROCESS( - RESULT_VARIABLE INSTALL_STATUS - COMMAND /usr/bin/env ${XROOTD_PYBUILD_ENV} ${Python_EXECUTABLE} ${SETUP_PY} install \ - --verbose \ - --prefix \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX} \ - ${DEB_INSTALL_ARGS} - ) - if(NOT INSTALL_STATUS EQUAL 0) - message(FATAL_ERROR \"Failed to install Python bindings\") - endif() - ") -else() - install( - CODE - "EXECUTE_PROCESS( - RESULT_VARIABLE INSTALL_STATUS - COMMAND /usr/bin/env ${XROOTD_PYBUILD_ENV} ${Python_EXECUTABLE} -m pip install \ - ${PIP_OPTIONS} \ - ${CMAKE_CURRENT_BINARY_DIR} - ) + install(CODE " + execute_process(COMMAND ${Python_EXECUTABLE} -m pip install ${PIP_OPTIONS} + --prefix \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX} ${CMAKE_CURRENT_BINARY_DIR} + RESULT_VARIABLE INSTALL_STATUS COMMAND_ECHO STDOUT) if(NOT INSTALL_STATUS EQUAL 0) message(FATAL_ERROR \"Failed to install Python bindings\") endif() - ") + ") + endif() endif() diff --git a/bindings/python/MANIFEST.in b/bindings/python/MANIFEST.in index 2110b2157d3..2dd8b3ebfb0 100644 --- a/bindings/python/MANIFEST.in +++ b/bindings/python/MANIFEST.in @@ -1,5 +1,4 @@ -include README.rst -include VERSION_INFO +include CMakeLists.txt recursive-include tests * recursive-include examples *.py recursive-include docs * diff --git a/bindings/python/README.md b/bindings/python/README.md new file mode 100644 index 00000000000..e46add4581c --- /dev/null +++ b/bindings/python/README.md @@ -0,0 +1,6 @@ +## XRootD Python Bindings + +This is a set of simple but pythonic bindings for XRootD. It is designed to make +it easy to interface with the XRootD client, by writing Python instead of having +to write C++. + diff --git a/bindings/python/README.rst b/bindings/python/README.rst deleted file mode 100644 index cf5c2bd0129..00000000000 --- a/bindings/python/README.rst +++ /dev/null @@ -1,16 +0,0 @@ -Prerequisites (incomplete): xrootd - -# Installing on OSX -Setup should succeed if: - - xrootd is installed on your system - - xrootd was installed via homebrew - - you're installing the bindings package with the same version number as the - xrootd installation (`xrootd -v`). - -If you have xrootd installed and the installation still fails, do -`XRD_LIBDIR=XYZ; XRD_INCDIR=ZYX; pip install xrootd` -where XYZ and ZYX are the paths to the XRootD library and include directories on your system. - -## How to find the lib and inc directories -To find the library directory, search your system for "libXrd*" files. -The include directory should contain a file named "XrdVersion.hh", so search for that. diff --git a/bindings/python/VERSION b/bindings/python/VERSION new file mode 120000 index 00000000000..558194c5a5a --- /dev/null +++ b/bindings/python/VERSION @@ -0,0 +1 @@ +../../VERSION \ No newline at end of file diff --git a/bindings/python/docs/source/conf.py b/bindings/python/docs/source/conf.py index 14aa9782860..1798e88aafa 100644 --- a/bindings/python/docs/source/conf.py +++ b/bindings/python/docs/source/conf.py @@ -240,4 +240,4 @@ #texinfo_show_urls = 'footnote' # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} +intersphinx_mapping = { 'python': ('https://docs.python.org/3/', None) } diff --git a/bindings/python/docs/source/install.rst b/bindings/python/docs/source/install.rst index b382321edde..54c5bab7101 100644 --- a/bindings/python/docs/source/install.rst +++ b/bindings/python/docs/source/install.rst @@ -1,6 +1,224 @@ -=========================== -**Installing** ``pyxrootd`` -=========================== +================================= +Installing XRootD Python Bindings +================================= -.. include:: ../../README.rst - :start-line: 8 +For general instructions on how to use ``pip`` to install Python packages, please +take a look at https://packaging.python.org/en/latest/tutorials/installing-packages/. +The installation of XRootD and its Python bindings follows for the most part the +same procedure. However, there are some important things that are specific to +XRootD, which we discuss here. Since XRootD 5.6, it is possible to use ``pip`` to +install only the Python bindings, building it against a pre-installed version of +XRootD. In this case, we recommend using the same version of XRootD for both +parts, even if the newer Python bindings should be usable with older versions of +XRootD 5.x. Suppose that XRootD is installed already into ``/usr``. Then, one can +build and install the Python bindings as shown below:: + + xrootd $ cd bindings/python + python $ python3 -m pip install --target install/ . + Processing xrootd/bindings/python + Installing build dependencies ... done + Getting requirements to build wheel ... done + Installing backend dependencies ... done + Preparing metadata (pyproject.toml) ... done + Building wheels for collected packages: xrootd + Building wheel for xrootd (pyproject.toml) ... done + Created wheel for xrootd: filename=xrootd-5.6-cp311-cp311-linux_x86_64.whl size=203460 sha256=8bbd9168... + Stored in directory: /tmp/pip-ephem-wheel-cache-rc_kb_nx/wheels/af/1b/42/bb953908... + Successfully built xrootd + Installing collected packages: xrootd + +The command above installs the Python bindings into the ``install/`` directory in +the current working directory. The structure is as shown below:: + + install/ + |-- XRootD + | |-- __init__.py + | |-- __pycache__ + | | |-- __init__.cpython-311.pyc + | |-- client + | |-- __init__.py + | |-- __pycache__ + | | |-- __init__.cpython-311.pyc + | | |-- _version.cpython-311.pyc + | | |-- copyprocess.cpython-311.pyc + | | |-- env.cpython-311.pyc + | | |-- file.cpython-311.pyc + | | |-- filesystem.cpython-311.pyc + | | |-- finalize.cpython-311.pyc + | | |-- flags.cpython-311.pyc + | | |-- glob_funcs.cpython-311.pyc + | | |-- responses.cpython-311.pyc + | | |-- url.cpython-311.pyc + | | |-- utils.cpython-311.pyc + | |-- _version.py + | |-- copyprocess.py + | |-- env.py + | |-- file.py + | |-- filesystem.py + | |-- finalize.py + | |-- flags.py + | |-- glob_funcs.py + | |-- responses.py + | |-- url.py + | |-- utils.py + |-- pyxrootd + | |-- __init__.py + | |-- __pycache__ + | | |-- __init__.cpython-311.pyc + | |-- client.cpython-311-x86_64-linux-gnu.so + |-- xrootd-5.6.dist-info + |-- INSTALLER + |-- METADATA + |-- RECORD + |-- REQUESTED + |-- WHEEL + |-- direct_url.json + |-- top_level.txt + + 8 directories, 36 files + +If you would like to install it for your own user, then use +``pip install --user`` instead of ``--target``. + +If XRootD is not already installed into the system, then you will want to +install both the client libraries and the Python bindings together using ``pip``. +This is possible by using the ``setup.py`` at the top level of the project, rather +than the one in the ``bindings/python`` subdirectory:: + + xrootd $ python3 -m pip install --target install/ . + Processing xrootd + Installing build dependencies ... done + Getting requirements to build wheel ... done + Installing backend dependencies ... done + Preparing metadata (pyproject.toml) ... done + Building wheels for collected packages: xrootd + Building wheel for xrootd (pyproject.toml) ... done + Created wheel for xrootd: filename=xrootd-5.6-cp311-cp311-linux_x86_64.whl size=65315683 sha256=a2e7ff52... + Stored in directory: /tmp/pip-ephem-wheel-cache-9g6ovy4q/wheels/47/93/fc/a23666d3... + Successfully built xrootd + Installing collected packages: xrootd + Successfully installed xrootd-5.6 + +In this case, the structure is a bit different than before:: + + xrootd $ tree install/ + install/ + |-- XRootD + | |-- __init__.py + | |-- __pycache__ + | | |-- __init__.cpython-311.pyc + | |-- client + | |-- __init__.py + | |-- __pycache__ + | | |-- __init__.cpython-311.pyc + | | |-- _version.cpython-311.pyc + | | |-- copyprocess.cpython-311.pyc + | | |-- env.cpython-311.pyc + | | |-- file.cpython-311.pyc + | | |-- filesystem.cpython-311.pyc + | | |-- finalize.cpython-311.pyc + | | |-- flags.cpython-311.pyc + | | |-- glob_funcs.cpython-311.pyc + | | |-- responses.cpython-311.pyc + | | |-- url.cpython-311.pyc + | | |-- utils.cpython-311.pyc + | |-- _version.py + | |-- copyprocess.py + | |-- env.py + | |-- file.py + | |-- filesystem.py + | |-- finalize.py + | |-- flags.py + | |-- glob_funcs.py + | |-- responses.py + | |-- url.py + | |-- utils.py + |-- pyxrootd + | |-- __init__.py + | |-- __pycache__ + | | |-- __init__.cpython-311.pyc + | |-- client.cpython-311-x86_64-linux-gnu.so + | |-- libXrdAppUtils.so + | |-- libXrdAppUtils.so.2 + | |-- libXrdAppUtils.so.2.0.0 + | |-- libXrdCl.so + | |-- libXrdCl.so.3 + | |-- libXrdCl.so.3.0.0 + | |-- libXrdClHttp-5.so + | |-- libXrdClProxyPlugin-5.so + | |-- libXrdClRecorder-5.so + | |-- libXrdCrypto.so + | |-- libXrdCrypto.so.2 + | |-- libXrdCrypto.so.2.0.0 + | |-- libXrdCryptoLite.so + | |-- libXrdCryptoLite.so.2 + | |-- libXrdCryptoLite.so.2.0.0 + | |-- libXrdCryptossl-5.so + | |-- libXrdPosix.so + | |-- libXrdPosix.so.3 + | |-- libXrdPosix.so.3.0.0 + | |-- libXrdPosixPreload.so + | |-- libXrdPosixPreload.so.2 + | |-- libXrdPosixPreload.so.2.0.0 + | |-- libXrdSec-5.so + | |-- libXrdSecProt-5.so + | |-- libXrdSecgsi-5.so + | |-- libXrdSecgsiAUTHZVO-5.so + | |-- libXrdSecgsiGMAPDN-5.so + | |-- libXrdSeckrb5-5.so + | |-- libXrdSecpwd-5.so + | |-- libXrdSecsss-5.so + | |-- libXrdSecunix-5.so + | |-- libXrdSecztn-5.so + | |-- libXrdUtils.so + | |-- libXrdUtils.so.3 + | |-- libXrdUtils.so.3.0.0 + | |-- libXrdXml.so + | |-- libXrdXml.so.3 + | |-- libXrdXml.so.3.0.0 + |-- xrootd-5.6.dist-info + |-- COPYING + |-- COPYING.BSD + |-- COPYING.LGPL + |-- INSTALLER + |-- LICENSE + |-- METADATA + |-- RECORD + |-- REQUESTED + |-- WHEEL + |-- direct_url.json + |-- top_level.txt + + 8 directories, 78 files + +As can be seen above, now all client libraries have been installed alongside the +C++ Python bindings library (``client.cpython-311-x86_64-linux-gnu.so``). When +installing via ``pip`` by simply calling ``pip install xrootd``, the package that +gets installed is in this mode which includes the libraries. However, command +line tools are not included. + +Binary wheels are supported as well. They can be built using the ``wheel`` +subcommand instead of ``install``:: + + xrootd $ python3.12 -m pip wheel . + Processing xrootd + Installing build dependencies ... done + Getting requirements to build wheel ... done + Installing backend dependencies ... done + Preparing metadata (pyproject.toml) ... done + Building wheels for collected packages: xrootd + Building wheel for xrootd (pyproject.toml) ... done + Created wheel for xrootd: filename=xrootd-5.6-cp312-cp312-linux_x86_64.whl size=65318541 sha256=6c4ed389... + Stored in directory: /tmp/pip-ephem-wheel-cache-etujwyx1/wheels/cf/67/3c/514b21dd... + Successfully built xrootd + +If you want to have everything installed, that is, server, client, command line +tools, etc, then it is recommended to use CMake to build the project, and use +the options ``-DENABLE_PYTHON=ON -DINSTALL_PYTHON_BINDINGS=ON`` so that CMake +takes care of calling ``pip`` to install the Python bindings compiled together +with the other components in the end. The option ``-DPIP_OPTIONS`` can be used to +pass on options to pip, but it should never be used to change the installation +prefix, as that is handled by CMake. Please see INSTALL.md_ for instructions on +how to build XRootD from source using CMake. + +.. _INSTALL.md: https://github.com/xrootd/xrootd/blob/master/docs/INSTALL.md diff --git a/bindings/python/libs/client/file.py b/bindings/python/libs/client/file.py index bff9b429e7f..fe44df570a0 100644 --- a/bindings/python/libs/client/file.py +++ b/bindings/python/libs/client/file.py @@ -44,10 +44,11 @@ def __iter__(self): return self def __next__(self): - return self.__file.next() + return self.__file.__next__() # Python 2 compatibility - next = __next__ + def next(self): + return self.__file.next() def open(self, url, flags=0, mode=0, timeout=0, callback=None): """Open the file pointed to by the given URL. diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml new file mode 120000 index 00000000000..00c904eb842 --- /dev/null +++ b/bindings/python/pyproject.toml @@ -0,0 +1 @@ +../../pyproject.toml \ No newline at end of file diff --git a/bindings/python/setup.py b/bindings/python/setup.py new file mode 100644 index 00000000000..737f2d6bd53 --- /dev/null +++ b/bindings/python/setup.py @@ -0,0 +1,155 @@ +import os +import platform +import subprocess +import sys + +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext +from subprocess import check_call, check_output + +try: + from shutil import which +except ImportError: + from distutils.spawn import find_executable as which + +def get_cmake_args(): + args = os.getenv('CMAKE_ARGS') + + if not args: + return [] + + from shlex import split + return split(args) + +srcdir = '${CMAKE_CURRENT_SOURCE_DIR}' + +cmdline_args = [] + +# Check for unexpanded srcdir to determine if this is part +# of a regular CMake build or a Python build using setup.py. + +if not srcdir.startswith('$'): + # When building the Python bindings as part of a standard CMake build, + # propagate down which cmake command to use, and the build type, C++ + # compiler, build flags, and how to link libXrdCl from the main build. + + cmake = '${CMAKE_COMMAND}' + + cmdline_args += [ + '-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}', + '-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}', + '-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}', + '-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}', + '-DXRootD_CLIENT_LIBRARY=${CMAKE_BINARY_DIR}/src/XrdCl/libXrdCl${CMAKE_SHARED_LIBRARY_SUFFIX}', + '-DXRootD_INCLUDE_DIR=${CMAKE_SOURCE_DIR}/src;${CMAKE_BINARY_DIR}/src', + ] +else: + srcdir = '.' + + cmake = which("cmake3") or which("cmake") + + cmdline_args += get_cmake_args() + +def get_version(): + version = '${XRootD_VERSION_STRING}' + + if version.startswith('$'): + try: + version = open('VERSION').read().strip() + + if version.startswith('$'): + output = check_output(['git', 'describe']) + version = output.decode().strip() + except: + version = None + pass + + if version is None: + from datetime import date + version = '5.7-rc' + date.today().strftime("%Y%m%d") + + if version.startswith('v'): + version = version[1:] + + # Sanitize version to conform to PEP 440 + # https://www.python.org/dev/peps/pep-0440 + version = version.replace('-rc', 'rc') + version = version.replace('-g', '+git.') + version = version.replace('-', '.post', 1) + version = version.replace('-', '.') + + return version + +class CMakeExtension(Extension): + def __init__(self, name, src=srcdir, sources=[], **kwa): + Extension.__init__(self, name, sources=sources, **kwa) + self.src = os.path.abspath(src) + +class CMakeBuild(build_ext): + def build_extensions(self): + if cmake is None: + raise RuntimeError('Cannot find CMake executable') + + for ext in self.extensions: + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + + # Use relative RPATHs to ensure the correct libraries are picked up. + # The RPATH below covers most cases where a non-standard path is + # used for installation. It allows to find libXrdCl with a relative + # path from the site-packages directory. Build with install RPATH + # because libraries are installed by Python/pip not CMake, so CMake + # cannot fix the install RPATH later on. + + cmake_args = [ + '-DPython_EXECUTABLE={}'.format(sys.executable), + '-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE', + '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}/{}'.format(self.build_temp, ext.name), + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}/{}'.format(extdir, ext.name), + ] + + if sys.platform == 'darwin': + cmake_args += [ '-DCMAKE_INSTALL_RPATH=@loader_path/../../..' ] + else: + cmake_args += [ '-DCMAKE_INSTALL_RPATH=$ORIGIN/../../../../$LIB' ] + + cmake_args += cmdline_args + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + check_call([cmake, ext.src, '-B', self.build_temp] + cmake_args) + check_call([cmake, '--build', self.build_temp, '--parallel']) + +version = get_version() + +setup(name='xrootd', + version=version, + description='XRootD Python bindings', + author='XRootD Developers', + author_email='xrootd-dev@slac.stanford.edu', + url='http://xrootd.org', + download_url='https://github.com/xrootd/xrootd/archive/v%s.tar.gz' % version, + keywords=['XRootD', 'network filesystem'], + license='LGPLv3+', + long_description=open(srcdir + '/README.md').read(), + long_description_content_type='text/plain', + packages = ['XRootD', 'XRootD.client', 'pyxrootd'], + package_dir = { + 'pyxrootd' : srcdir + '/src', + 'XRootD' : srcdir + '/libs', + 'XRootD/client': srcdir + '/libs/client', + }, + ext_modules= [ CMakeExtension('pyxrootd') ], + cmdclass={ 'build_ext': CMakeBuild }, + zip_safe=False, + classifiers=[ + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + "Operating System :: Unix", + "Programming Language :: C++", + "Programming Language :: Python", + ] + ) diff --git a/bindings/python/setup.py.in b/bindings/python/setup.py.in deleted file mode 100644 index 7fb8c53f63d..00000000000 --- a/bindings/python/setup.py.in +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import print_function - -from setuptools import setup, Extension -# sysconfig with setuptools v48.0.0+ is incompatible for Python 3.6 only, so fall back to distutils. -# FIXME: When support for Python 3.6 is dropped simplify this -import sys - -if sys.version_info < (3, 7): - from distutils import sysconfig -else: - import sysconfig - -from os import getenv, walk, path -import subprocess - -# Remove the "-Wstrict-prototypes" compiler option, which isn't valid for C++. -cfg_vars = sysconfig.get_config_vars() -opt = cfg_vars["OPT"] -cfg_vars["OPT"] = " ".join( flag for flag in opt.split() if flag not in ['-Wstrict-prototypes' ${CLANG_PROHIBITED} ] ) + " --std=c++14" - -cflags = cfg_vars["CFLAGS"] -cfg_vars["CFLAGS"] = " ".join( flag for flag in cflags.split() if flag not in ['-Wstrict-prototypes' ${CLANG_PROHIBITED} ] ) + " --std=c++14" - -# pypy doesn't define PY_CFLAGS so skip it if it's missing -if "PY_CFLAGS" in cfg_vars: - py_cflags = cfg_vars["PY_CFLAGS"] - cfg_vars["PY_CFLAGS"] = " ".join( flag for flag in py_cflags.split() if flag not in ['-Wstrict-prototypes' ${CLANG_PROHIBITED} ] ) + " --std=c++14" - -ccl=cfg_vars["CC"].split() -ccl[0]="${CMAKE_C_COMPILER}" -cfg_vars["CC"] = " ".join(ccl) -cxxl=cfg_vars["CXX"].split() -cxxl[0]="${CMAKE_CXX_COMPILER}" -cfg_vars["CXX"] = " ".join(cxxl) -cfg_vars["PY_CXXFLAGS"] = "${CMAKE_CXX_FLAGS}" - -# Make the RPATH relative to the python module -cfg_vars['LDSHARED'] = cfg_vars['LDSHARED'] + " -Wl,-rpath,${XRDCL_RPATH}" - -sources = list() -depends = list() - -for dirname, dirnames, filenames in walk('${CMAKE_CURRENT_SOURCE_DIR}/src'): - for filename in filenames: - if filename.endswith('.cc'): - sources.append(path.join(dirname, filename)) - elif filename.endswith('.hh'): - depends.append(path.join(dirname, filename)) - -xrdcllibdir = "${XRDCL_LIBDIR}" -xrdlibdir = "${XRD_LIBDIR}" -xrdsrcincdir = "${XRD_SRCINCDIR}" -xrdbinincdir = "${XRD_BININCDIR}" -version = "${XROOTD_VERSION}" - -if version.startswith('unknown'): - try: - import os - version_file_path = os.path.join('${CMAKE_CURRENT_SOURCE_DIR}', 'VERSION') - print('Version file path: {}'.format(version_file_path)) - with open(version_file_path, 'r') as f: - version = f.read().split('/n')[0] - print('Version from file: {}'.format(version)) - except Exception as e: - print('{} \nCannot open VERSION_INFO file. {} will be used'.format(e, version)) - -# Sanitize in keeping with PEP 440 -# c.f. https://www.python.org/dev/peps/pep-0440/ -# c.f. https://github.com/pypa/pip/issues/8368 -# version needs to pass pip._vendor.packaging.version.Version() -version = version.replace("-", ".") - -if version.startswith("v"): - version = version[1:] - -version_parts = version.split(".") - -# Ensure release candidates sanitized to ..rc -if version_parts[-1].startswith("rc"): - version = ".".join(version_parts[:-1]) + version_parts[-1] - version_parts = version.split(".") - -if len(version_parts[0]) == 8: - # CalVer - date = version_parts[0] - year = date[:4] - incremental = date[4:] - if incremental.startswith("0"): - incremental = incremental[1:] - - version = year + "." + incremental - - if len(version_parts) > 1: - # https://github.com/pypa/pip/issues/9188#issuecomment-736025963 - hash = version_parts[1] - version = version + "+" + hash - -print('XRootD library dir: ', xrdlibdir) -print('XRootD src include dir:', xrdsrcincdir) -print('XRootD bin include dir:', xrdbinincdir) -print('Version: ', version) - -setup( name = 'xrootd', - version = version, - author = 'XRootD Developers', - author_email = 'xrootd-dev@slac.stanford.edu', - url = 'http://xrootd.org', - license = 'LGPLv3+', - description = "XRootD Python bindings", - long_description = "XRootD Python bindings", - packages = ['pyxrootd', 'XRootD', 'XRootD.client'], - package_dir = {'pyxrootd' : '${CMAKE_CURRENT_SOURCE_DIR}/src', - 'XRootD' : '${CMAKE_CURRENT_SOURCE_DIR}/libs', - 'XRootD.client': '${CMAKE_CURRENT_SOURCE_DIR}/libs/client'}, - ext_modules = [ - Extension( - 'pyxrootd.client', - sources = sources, - depends = depends, - libraries = ['XrdCl', 'XrdUtils', 'dl'], - extra_compile_args = ['-g'], - include_dirs = [xrdsrcincdir, xrdbinincdir], - library_dirs = [xrdlibdir, xrdcllibdir] - ) - ] - ) diff --git a/bindings/python/src/CMakeLists.txt b/bindings/python/src/CMakeLists.txt new file mode 100644 index 00000000000..5ea0e703445 --- /dev/null +++ b/bindings/python/src/CMakeLists.txt @@ -0,0 +1,63 @@ +# WITH_SOABI was introduced in CMake 3.17 +# https://cmake.org/cmake/help/latest/module/FindPython.html#commands +if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.17) + set(SOABI WITH_SOABI) +endif() + +Python_add_library(client MODULE ${SOABI} + # headers + AsyncResponseHandler.hh + ChunkIterator.hh + Conversions.hh + PyXRootD.hh + PyXRootDCopyProcess.hh + PyXRootDCopyProgressHandler.hh + PyXRootDEnv.hh + PyXRootDFile.hh + PyXRootDFileSystem.hh + PyXRootDFinalize.hh + PyXRootDURL.hh + Utils.hh + # sources + PyXRootDCopyProcess.cc + PyXRootDCopyProgressHandler.cc + PyXRootDFile.cc + PyXRootDFileSystem.cc + PyXRootDModule.cc + PyXRootDURL.cc + Utils.cc +) + +target_compile_options(client PRIVATE -w) # TODO: fix build warnings + +if(APPLE) + set(CMAKE_MACOSX_RPATH TRUE) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + set_target_properties(client PROPERTIES INSTALL_NAME_DIR "@rpath") +endif() + +# Avoid a call to find_package(XRootD) in order to be able to override +# variables when building the module as part of a standard CMake build. + +if(TARGET XrdCl) + target_link_libraries(client PRIVATE XrdCl) +else() + find_library(XRootD_CLIENT_LIBRARY NAMES XrdCl) + + if(NOT XRootD_CLIENT_LIBRARY) + message(FATAL_ERROR "Could not find XRootD client library") + endif() + + find_path(XRootD_INCLUDE_DIR XrdVersion.hh PATH_SUFFIXES include/xrootd) + + if(NOT XRootD_INCLUDE_DIR) + message(FATAL_ERROR "Could not find XRootD client include directory") + endif() + + # The client library makes use of private XRootD headers, so add the + # extra include for it to allow building the Python bindings against + # a pre-installed XRootD. + + target_link_libraries(client PRIVATE ${XRootD_CLIENT_LIBRARY}) + target_include_directories(client PRIVATE ${XRootD_INCLUDE_DIR} ${XRootD_INCLUDE_DIR}/private) +endif() diff --git a/bindings/python/src/PyXRootD.hh b/bindings/python/src/PyXRootD.hh index a979186865a..054f9852676 100644 --- a/bindings/python/src/PyXRootD.hh +++ b/bindings/python/src/PyXRootD.hh @@ -32,6 +32,13 @@ #if PY_MAJOR_VERSION >= 3 #define IS_PY3K +#define Py_TPFLAGS_HAVE_ITER 0 +#else +#define PyUnicode_AsUTF8 PyString_AsString +#define PyUnicode_Check PyString_Check +#define PyUnicode_FromString PyString_FromString +#define PyUnicode_FromStringAndSize PyString_FromStringAndSize +#define PyUnicode_GET_LENGTH PyString_Size #endif #define async( func ) \ @@ -39,16 +46,4 @@ func; \ Py_END_ALLOW_THREADS \ -#ifdef IS_PY3K -#define Py_TPFLAGS_HAVE_ITER 0 -#else -#if PY_MINOR_VERSION <= 5 -#define PyUnicode_FromString PyString_FromString -#endif -#define PyBytes_Size PyString_Size -#define PyBytes_Check PyString_Check -#define PyBytes_FromString PyString_FromString -#define PyBytes_FromStringAndSize PyString_FromStringAndSize -#endif - #endif /* PYXROOTD_HH_ */ diff --git a/bindings/python/src/PyXRootDEnv.hh b/bindings/python/src/PyXRootDEnv.hh index 5642a86a0e9..3c1357f6620 100644 --- a/bindings/python/src/PyXRootDEnv.hh +++ b/bindings/python/src/PyXRootDEnv.hh @@ -134,7 +134,7 @@ namespace PyXRootD //---------------------------------------------------------------------------- PyObject* XrdVersion_cpp( PyObject *self, PyObject *args ) { - static std::string verstr( XrdVERSION[0] == 'v' ? XrdVERSION + 1 : XrdVERSION ); + static std::string verstr( XrdVERSION[0] == 'v' ? &XrdVERSION[1] : XrdVERSION ); return Py_BuildValue( "s", verstr.c_str() ); } diff --git a/bindings/python/src/PyXRootDFile.cc b/bindings/python/src/PyXRootDFile.cc index 619100990dd..09ebbea0d39 100644 --- a/bindings/python/src/PyXRootDFile.cc +++ b/bindings/python/src/PyXRootDFile.cc @@ -106,7 +106,7 @@ namespace PyXRootD PyObject* File::Stat( File *self, PyObject *args, PyObject *kwds ) { static const char *kwlist[] = { "force", "timeout", "callback", NULL }; - bool force = false; + int force = 0; uint16_t timeout = 0; PyObject *callback = NULL, *pyresponse = NULL, *pystatus = NULL; XrdCl::XRootDStatus status; @@ -295,10 +295,10 @@ namespace PyXRootD if ( off_init == 0 ) self->currentOffset += line->GetSize(); - pyline = PyBytes_FromStringAndSize( line->GetBuffer(), line->GetSize() ); + pyline = PyUnicode_FromStringAndSize( line->GetBuffer(), line->GetSize() ); } else - pyline = PyBytes_FromString( "" ); + pyline = PyUnicode_FromString( "" ); delete line; delete chunk; @@ -346,7 +346,7 @@ namespace PyXRootD { line = self->ReadLine( self, args, kwds ); - if ( !line || PyBytes_Size( line ) == 0 ) + if ( !line || PyUnicode_GET_LENGTH( line ) == 0 ) break; PyList_Append( lines, line ); @@ -800,14 +800,14 @@ namespace PyXRootD return NULL; // extract the attribute name from the tuple PyObject *py_name = PyTuple_GetItem( item, 0 ); - if( !PyBytes_Check( py_name ) ) + if( !PyUnicode_Check( py_name ) ) return NULL; - std::string name = PyBytes_AsString( py_name ); + std::string name = PyUnicode_AsUTF8( py_name ); // extract the attribute value from the tuple PyObject *py_value = PyTuple_GetItem( item, 1 ); - if( !PyBytes_Check( py_value ) ) + if( !PyUnicode_Check( py_value ) ) return NULL; - std::string value = PyBytes_AsString( py_value ); + std::string value = PyUnicode_AsUTF8( py_value ); // update the C++ list of xattrs attrs.push_back( XrdCl::xattr_t( name, value ) ); } @@ -864,9 +864,9 @@ namespace PyXRootD // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a string - if( !item || !PyBytes_Check( item ) ) + if( !item || !PyUnicode_Check( item ) ) return NULL; - std::string name = PyBytes_AsString( item ); + std::string name = PyUnicode_AsUTF8( item ); // update the C++ list of xattrs attrs.push_back( name ); } @@ -923,9 +923,9 @@ namespace PyXRootD // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a string - if( !item || !PyBytes_Check( item ) ) + if( !item || !PyUnicode_Check( item ) ) return NULL; - std::string name = PyBytes_AsString( item ); + std::string name = PyUnicode_AsUTF8( item ); // update the C++ list of xattrs attrs.push_back( name ); } diff --git a/bindings/python/src/PyXRootDFile.hh b/bindings/python/src/PyXRootDFile.hh index 8ce054b3d3c..bdc7568b823 100644 --- a/bindings/python/src/PyXRootDFile.hh +++ b/bindings/python/src/PyXRootDFile.hh @@ -124,7 +124,7 @@ namespace PyXRootD //-------------------------------------------------------------------------- // Raise StopIteration if the line we just read is empty //-------------------------------------------------------------------------- - if ( PyBytes_Size( line ) == 0 ) { + if ( PyUnicode_GET_LENGTH( line ) == 0 ) { PyErr_SetNone( PyExc_StopIteration ); return NULL; } diff --git a/bindings/python/src/PyXRootDFileSystem.cc b/bindings/python/src/PyXRootDFileSystem.cc index ceb2c01f7a6..4eb1c8a831f 100644 --- a/bindings/python/src/PyXRootDFileSystem.cc +++ b/bindings/python/src/PyXRootDFileSystem.cc @@ -625,15 +625,16 @@ namespace PyXRootD } std::vector files; - const char *file; - PyObject *pyfile; + for (int i = 0; i < PyList_Size(pyfiles); ++i) { + PyObject *item = PyList_GetItem(pyfiles, i); - // Convert python list to stl vector - for ( int i = 0; i < PyList_Size( pyfiles ); ++i ) { - pyfile = PyList_GetItem( pyfiles, i ); - if ( !pyfile ) return NULL; - file = PyBytes_AsString( pyfile ); - files.push_back( std::string( file ) ); + if (!PyUnicode_Check(item)) { + PyErr_SetString(PyExc_TypeError, + "files parameter must be a list of strings"); + return NULL; + } + + files.emplace_back(PyUnicode_AsUTF8(item)); } XrdCl::PrepareFlags::Flags flags; @@ -769,14 +770,14 @@ namespace PyXRootD return NULL; // extract the attribute name from the tuple PyObject *py_name = PyTuple_GetItem( item, 0 ); - if( !PyBytes_Check( py_name ) ) + if( !PyUnicode_Check( py_name ) ) return NULL; - std::string name = PyBytes_AsString( py_name ); + std::string name = PyUnicode_AsUTF8( py_name ); // extract the attribute value from the tuple PyObject *py_value = PyTuple_GetItem( item, 1 ); - if( !PyBytes_Check( py_value ) ) + if( !PyUnicode_Check( py_value ) ) return NULL; - std::string value = PyBytes_AsString( py_value ); + std::string value = PyUnicode_AsUTF8( py_value ); // update the C++ list of xattrs attrs.push_back( XrdCl::xattr_t( name, value ) ); } @@ -831,9 +832,9 @@ namespace PyXRootD // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a string - if( !item || !PyBytes_Check( item ) ) + if( !item || !PyUnicode_Check( item ) ) return NULL; - std::string name = PyBytes_AsString( item ); + std::string name = PyUnicode_AsUTF8( item ); // update the C++ list of xattrs attrs.push_back( name ); } @@ -888,9 +889,9 @@ namespace PyXRootD // get the item at respective index PyObject *item = PyList_GetItem( pyattrs, i ); // make sure the item is a string - if( !item || !PyBytes_Check( item ) ) + if( !item || !PyUnicode_Check( item ) ) return NULL; - std::string name = PyBytes_AsString( item ); + std::string name = PyUnicode_AsUTF8( item ); // update the C++ list of xattrs attrs.push_back( name ); } diff --git a/bindings/python/src/PyXRootDURL.cc b/bindings/python/src/PyXRootDURL.cc index ee957258087..3552605999c 100644 --- a/bindings/python/src/PyXRootDURL.cc +++ b/bindings/python/src/PyXRootDURL.cc @@ -56,12 +56,12 @@ namespace PyXRootD //---------------------------------------------------------------------------- int URL::SetProtocol( URL *self, PyObject *protocol, void *closure ) { - if ( !PyBytes_Check( protocol ) ) { + if ( !PyUnicode_Check( protocol ) ) { PyErr_SetString( PyExc_TypeError, "protocol must be string" ); return -1; } - self->url->SetProtocol( std::string ( PyBytes_AsString( protocol ) ) ); + self->url->SetProtocol( std::string ( PyUnicode_AsUTF8( protocol ) ) ); return 0; } @@ -78,12 +78,12 @@ namespace PyXRootD //---------------------------------------------------------------------------- int URL::SetUserName( URL *self, PyObject *username, void *closure ) { - if ( !PyBytes_Check( username ) ) { + if ( !PyUnicode_Check( username ) ) { PyErr_SetString( PyExc_TypeError, "username must be string" ); return -1; } - self->url->SetUserName( std::string( PyBytes_AsString( username ) ) ); + self->url->SetUserName( std::string( PyUnicode_AsUTF8( username ) ) ); return 0; } @@ -100,12 +100,12 @@ namespace PyXRootD //---------------------------------------------------------------------------- int URL::SetPassword( URL *self, PyObject *password, void *closure ) { - if ( !PyBytes_Check( password ) ) { + if ( !PyUnicode_Check( password ) ) { PyErr_SetString( PyExc_TypeError, "password must be string" ); return -1; } - self->url->SetPassword( std::string( PyBytes_AsString( password ) ) ); + self->url->SetPassword( std::string( PyUnicode_AsUTF8( password ) ) ); return 0; } @@ -122,12 +122,12 @@ namespace PyXRootD //---------------------------------------------------------------------------- int URL::SetHostName( URL *self, PyObject *hostname, void *closure ) { - if ( !PyBytes_Check( hostname ) ) { + if ( !PyUnicode_Check( hostname ) ) { PyErr_SetString( PyExc_TypeError, "hostname must be string" ); return -1; } - self->url->SetHostName( std::string( PyBytes_AsString( hostname ) ) ); + self->url->SetHostName( std::string( PyUnicode_AsUTF8( hostname ) ) ); return 0; } @@ -178,12 +178,12 @@ namespace PyXRootD //---------------------------------------------------------------------------- int URL::SetPath( URL *self, PyObject *path, void *closure ) { - if ( !PyBytes_Check( path ) ) { + if ( !PyUnicode_Check( path ) ) { PyErr_SetString( PyExc_TypeError, "path must be string" ); return -1; } - self->url->SetPath( std::string( PyBytes_AsString( path ) ) ); + self->url->SetPath( std::string( PyUnicode_AsUTF8( path ) ) ); return 0; } diff --git a/cmake/FindDavix.cmake b/cmake/FindDavix.cmake index 8013d610d77..bd141c18478 100644 --- a/cmake/FindDavix.cmake +++ b/cmake/FindDavix.cmake @@ -1,23 +1,69 @@ -find_path( - Davix_INCLUDE_DIRS - NAMES davix/davix.hpp - HINTS ${Davix_INCLUDE_DIRS} -) +#.rst: +# FindDavix +# ------- +# +# Find Davix library for file management over HTTP-based protocols. +# +# Imported Targets +# ^^^^^^^^^^^^^^^^ +# +# This module defines :prop_tgt:`IMPORTED` target: +# +# ``Davix::Davix`` +# The libdavix library, if found. +# +# Result Variables +# ^^^^^^^^^^^^^^^^ +# +# This module will set the following variables in your project: +# +# ``DAVIX_FOUND`` +# True if Davix has been found. +# ``DAVIX_INCLUDE_DIRS`` +# Where to find davix.hpp, etc. +# ``DAVIX_LIBRARIES`` +# The libraries to link against to use Davix. +# ``DAVIX_VERSION`` +# The version of the Davix library found (e.g. 0.6.4) +# +# Obsolete variables +# ^^^^^^^^^^^^^^^^^^ +# +# The following variables may also be set, for backwards compatibility: +# +# ``DAVIX_LIBRARY`` +# where to find the DAVIX library. +# ``DAVIX_INCLUDE_DIR`` +# where to find the DAVIX headers (same as DAVIX_INCLUDE_DIRS) +# -find_library( - Davix_LIBRARIES - NAMES davix - HINTS ${Davix_LIBRARY_DIRS} -) +foreach(var FOUND INCLUDE_DIR INCLUDE_DIRS LIBRARY LIBRARIES) + unset(DAVIX_${var} CACHE) +endforeach() -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS( - Davix - DEFAULT_MSG - Davix_LIBRARIES - Davix_INCLUDE_DIRS -) +find_package(PkgConfig) -if(Davix_FOUND) - mark_as_advanced(Davix_LIBRARIES Davix_INCLUDE_DIRS) +if(PKG_CONFIG_FOUND) + if(${Davix_FIND_REQUIRED}) + set(Davix_REQUIRED REQUIRED) + endif() + + if(NOT DEFINED Davix_FIND_VERSION) + pkg_check_modules(DAVIX ${Davix_REQUIRED} davix) + else() + pkg_check_modules(DAVIX ${Davix_REQUIRED} davix>=${Davix_FIND_VERSION}) + endif() + + set(DAVIX_LIBRARIES ${DAVIX_LDFLAGS}) + set(DAVIX_LIBRARY ${DAVIX_LIBRARIES}) + set(DAVIX_INCLUDE_DIRS ${DAVIX_INCLUDE_DIRS}) + set(DAVIX_INCLUDE_DIR ${DAVIX_INCLUDE_DIRS}) endif() + +if(DAVIX_FOUND AND NOT TARGET Davix::Davix) + add_library(Davix::Davix INTERFACE IMPORTED) + set_property(TARGET Davix::Davix PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${DAVIX_INCLUDE_DIRS}") + set_property(TARGET Davix::Davix PROPERTY INTERFACE_LINK_LIBRARIES "${DAVIX_LIBRARIES}") +endif() + +mark_as_advanced(DAVIX_INCLUDE_DIR DAVIX_LIBRARY) diff --git a/cmake/FindSciTokensCpp.cmake b/cmake/FindSciTokensCpp.cmake index 3c9aabc91b5..1e37d3c64ea 100644 --- a/cmake/FindSciTokensCpp.cmake +++ b/cmake/FindSciTokensCpp.cmake @@ -1,4 +1,6 @@ +include(CheckSymbolExists) + FIND_PATH(SCITOKENS_CPP_INCLUDE_DIR scitokens/scitokens.h HINTS ${SCITOKENS_CPP_DIR} @@ -18,3 +20,9 @@ FIND_LIBRARY(SCITOKENS_CPP_LIBRARIES SciTokens INCLUDE(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(SciTokensCpp DEFAULT_MSG SCITOKENS_CPP_LIBRARIES SCITOKENS_CPP_INCLUDE_DIR) +IF (SCITOKENS_CPP_INCLUDE_DIR) + SET( CMAKE_REQUIRED_INCLUDES ${SCITOKENS_CPP_INCLUDE_DIR} ) + SET( CMAKE_REQUIRED_LIBRARIES ${SCITOKENS_CPP_LIBRARIES} ) + CHECK_SYMBOL_EXISTS(scitoken_config_set_str "scitokens/scitokens.h" HAVE_SCITOKEN_CONFIG_SET_STR) + MARK_AS_ADVANCED(HAVE_SCITOKEN_CONFIG_SET_STR) +ENDIF () diff --git a/cmake/Findlibuuid.cmake b/cmake/Findlibuuid.cmake index 7f07c4b53e0..6f44d1bd7a0 100644 --- a/cmake/Findlibuuid.cmake +++ b/cmake/Findlibuuid.cmake @@ -17,7 +17,7 @@ # # This module will set the following variables in your project: # -# ``UUID_FOUND`` +# ``LIBUUID_FOUND`` # True if libuuid has been found. # ``UUID_INCLUDE_DIRS`` # Where to find uuid/uuid.h. @@ -35,36 +35,79 @@ # where to find the uuid/uuid.h header (same as UUID_INCLUDE_DIRS). include(CheckCXXSymbolExists) -include(CheckLibraryExists) -include(FindPackageHandleStandardArgs) if(NOT UUID_INCLUDE_DIR) + set(CMAKE_FIND_FRAMEWORK LAST) find_path(UUID_INCLUDE_DIR uuid/uuid.h) endif() -if(EXISTS UUID_INCLUDE_DIR) - set(UUID_INCLUDE_DIRS ${UUID_INCLUDE_DIR}) - set(CMAKE_REQUIRED_INCLUDES ${UUID_INCLUDE_DIRS}) +if(IS_DIRECTORY "${UUID_INCLUDE_DIR}") + set(CMAKE_REQUIRED_INCLUDES ${UUID_INCLUDE_DIR}) check_cxx_symbol_exists("uuid_generate_random" "uuid/uuid.h" _uuid_header_only) + unset(CMAKE_REQUIRED_INCLUDES) endif() -if(NOT _uuid_header_only AND NOT UUID_LIBRARY) - check_library_exists("uuid" "uuid_generate_random" "" _have_libuuid) - if(_have_libuuid) - set(UUID_LIBRARY "uuid") - set(UUID_LIBRARIES ${UUID_LIBRARY}) +if(NOT UUID_LIBRARY AND NOT _uuid_header_only) + find_library(UUID_LIBRARY NAMES uuid) + + if(UUID_LIBRARY) + set(CMAKE_REQUIRED_INCLUDES ${UUID_INCLUDE_DIR}) + set(CMAKE_REQUIRED_LIBRARIES ${UUID_LIBRARY}) + check_cxx_symbol_exists("uuid_generate_random" "uuid/uuid.h" _have_libuuid) + unset(CMAKE_REQUIRED_INCLUDES) + unset(CMAKE_REQUIRED_LIBRARIES) + endif() + + if(NOT _have_libuuid) + find_package(PkgConfig QUIET) + if(PKG_CONFIG_FOUND) + # We need to clear cache variables set above, which pkg-config may set, + # otherwise the call to pkg_check_modules will have no effect, as it does + # not override cache variables. + foreach(var FOUND INCLUDE_DIR INCLUDE_DIRS LIBRARY LIBRARIES) + unset(UUID_${var} CACHE) + endforeach() + if(${libuuid_FIND_REQUIRED}) + set(libuuid_REQUIRED REQUIRED) + endif() + pkg_check_modules(UUID ${libuuid_REQUIRED} uuid) + + # The include directory returned by pkg-config is /include/uuid, + # while we expect just /include, so strip the last component to + # allow #include to actually work. + get_filename_component(UUID_INCLUDE_DIR ${UUID_INCLUDE_DIRS} DIRECTORY) + + set(UUID_INCLUDE_DIR ${UUID_INCLUDE_DIR} CACHE PATH "") + set(UUID_LIBRARY ${UUID_LDFLAGS} CACHE STRING "") + unset(UUID_INCLUDE_DIRS CACHE) + unset(UUID_LIBRARIES CACHE) + endif() endif() endif() -unset(CMAKE_REQUIRED_INCLUDES) -unset(_uuid_header_only) -unset(_have_libuuid) +if(_uuid_header_only) + find_package_handle_standard_args(libuuid DEFAULT_MSG UUID_INCLUDE_DIR) +else() + find_package_handle_standard_args(libuuid DEFAULT_MSG UUID_INCLUDE_DIR UUID_LIBRARY) +endif() -if(NOT TARGET uuid::uuid) - add_library(uuid::uuid INTERFACE IMPORTED) - set_property(TARGET uuid::uuid PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${UUID_INCLUDE_DIRS}") - set_property(TARGET uuid::uuid PROPERTY INTERFACE_LINK_LIBRARIES "${UUID_LIBRARIES}") +if(LIBUUID_FOUND) + set(UUID_INCLUDE_DIRS ${UUID_INCLUDE_DIR}) + set(UUID_LIBRARIES ${UUID_LIBRARY}) + if(NOT TARGET uuid::uuid) + add_library(uuid::uuid INTERFACE IMPORTED) + target_include_directories(uuid::uuid SYSTEM INTERFACE "${UUID_INCLUDE_DIRS}") + target_link_libraries(uuid::uuid INTERFACE "${UUID_LIBRARIES}") + endif() endif() -find_package_handle_standard_args(libuuid DEFAULT_MSG UUID_INCLUDE_DIR) mark_as_advanced(UUID_INCLUDE_DIR UUID_LIBRARY) + +if(NOT "${libuuid_FIND_QUIET}") + message(DEBUG "UUID_FOUND = ${LIBUUID_FOUND}") + message(DEBUG "UUID_HEADER_ONLY = ${_uuid_header_only}") + message(DEBUG "UUID_INCLUDE_DIR = ${UUID_INCLUDE_DIR}") + message(DEBUG "UUID_INCLUDE_DIRS = ${UUID_INCLUDE_DIRS}") + message(DEBUG "UUID_LIBRARY = ${UUID_LIBRARY}") + message(DEBUG "UUID_LIBRARIES = ${UUID_LIBRARIES}") +endif() diff --git a/cmake/XRootDCommon.cmake b/cmake/XRootDCommon.cmake deleted file mode 100644 index ec342c31657..00000000000 --- a/cmake/XRootDCommon.cmake +++ /dev/null @@ -1,7 +0,0 @@ - -if( READLINE_FOUND ) - include_directories( ${READLINE_INCLUDE_DIR} ) -endif() - -include_directories( ../ ./ ${ZLIB_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS} ${CMAKE_SOURCE_DIR}/src ${CMAKE_BINARY_DIR}/src - /usr/local/include ) diff --git a/cmake/XRootDConfig.cmake.in b/cmake/XRootDConfig.cmake.in index c70e94846ea..0a4ce3fe557 100644 --- a/cmake/XRootDConfig.cmake.in +++ b/cmake/XRootDConfig.cmake.in @@ -19,6 +19,20 @@ # List of components: CLIENT, UTILS, SERVER, POSIX, HTTP and SSI ################################################################################ +@PACKAGE_INIT@ + +set(XRootD_FOUND TRUE) + +set(XRootD_VERSION @XRootD_VERSION@) +set(XRootD_PLUGIN_VERSION @XRootD_VERSION_MAJOR@) + +set(XRootD_VERSION_MAJOR @XRootD_VERSION_MAJOR@) +set(XRootD_VERSION_MINOR @XRootD_VERSION_MINOR@) +set(XRootD_VERSION_PATCH @XRootD_VERSION_PATCH@) +set(XRootD_VERSION_TWEAK @XRootD_VERSION_TWEAK@) +set(XRootD_VERSION_NUMBER @XRootD_VERSION_NUMBER@) +set(XRootD_VERSION_STRING @XRootD_VERSION_STRING@) + ################################################################################ # Make sure all *_FOUND variables are intialized to FALSE ################################################################################ @@ -33,36 +47,21 @@ SET( XROOTD_SSI_FOUND FALSE ) ################################################################################ # Set XRootD include paths ################################################################################ -FIND_PATH( XROOTD_INCLUDE_DIRS XrdVersion.hh - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd - PATH_SUFFIXES include/xrootd - PATHS /opt/xrootd -) -IF( NOT "${XROOTD_INCLUDE_DIRS}" STREQUAL "XROOTD_INCLUDE_DIRS-NOTFOUND" ) - SET( XROOTD_FOUND TRUE ) - SET( XROOTD_INCLUDE_DIRS "${XROOTD_INCLUDE_DIRS};${XROOTD_INCLUDE_DIRS}/private" ) -ENDIF() +set_and_check(XRootD_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}") +set_and_check(XRootD_DATA_DIR "@PACKAGE_CMAKE_INSTALL_DATADIR@") +set_and_check(XRootD_INCLUDE_DIR "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@/xrootd") +set_and_check(XRootD_LIB_DIR "@PACKAGE_CMAKE_INSTALL_LIBDIR@") -IF( NOT XROOTD_FOUND ) - LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_FOUND ) -ENDIF() +set(XRootD_INCLUDE_DIRS "${XRootD_INCLUDE_DIR};${XRootD_INCLUDE_DIR}/private") +set(XROOTD_INCLUDE_DIRS "${XRootD_INCLUDE_DIRS}") # backward compatibility ################################################################################ # XRootD client libs # - libXrdCl ################################################################################ FIND_LIBRARY( XROOTD_CLIENT_LIBRARIES XrdCl - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd - PATH_SUFFIXES lib lib64 + PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_CLIENT_LIBRARIES}" STREQUAL "XROOTD_CLIENT_LIBRARIES-NOTFOUND" ) @@ -70,23 +69,12 @@ IF( NOT "${XROOTD_CLIENT_LIBRARIES}" STREQUAL "XROOTD_CLIENT_LIBRARIES-NOTFOUND" LIST( APPEND XROOTD_LIBRARIES ${XROOTD_CLIENT_LIBRARIES} ) ENDIF() -IF( XRootD_FIND_REQUIRED_CLIENT AND NOT XROOTD_CLIENT_FOUND ) - MESSAGE( "XRootD client required but not found!" ) - LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_CLIENT_FOUND ) - SET( XROOTD_FOUND FALSE ) -ENDIF() - ################################################################################ # XRootD utils libs # - libXrdUtils ################################################################################ FIND_LIBRARY( XROOTD_UTILS_LIBRARIES XrdUtils - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd - PATH_SUFFIXES lib lib64 + PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_UTILS_LIBRARIES}" STREQUAL "XROOTD_UTILS_LIBRARIES-NOTFOUND" ) @@ -94,23 +82,12 @@ IF( NOT "${XROOTD_UTILS_LIBRARIES}" STREQUAL "XROOTD_UTILS_LIBRARIES-NOTFOUND" ) LIST( APPEND XROOTD_LIBRARIES ${XROOTD_UTILS_LIBRARIES} ) ENDIF() -IF( XRootD_FIND_REQUIRED_UTILS AND NOT XROOTD_UTILS_FOUND ) - MESSAGE( "XRootD utils required but not found!" ) - LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_UTILS_FOUND ) - SET( XROOTD_FOUND FALSE ) -ENDIF() - ################################################################################ # XRootD server libs # - libXrdServer ################################################################################ FIND_LIBRARY( XROOTD_SERVER_LIBRARIES XrdServer - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd - PATH_SUFFIXES lib lib64 + PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_SERVER_LIBRARIES}" STREQUAL "XROOTD_SERVER_LIBRARIES-NOTFOUND" ) @@ -118,33 +95,17 @@ IF( NOT "${XROOTD_SERVER_LIBRARIES}" STREQUAL "XROOTD_SERVER_LIBRARIES-NOTFOUND" LIST( APPEND XROOTD_LIBRARIES ${XROOTD_SERVER_LIBRARIES} ) ENDIF() -IF( XRootD_FIND_REQUIRED_SERVER AND NOT XROOTD_SERVER_FOUND ) - MESSAGE( "XRootD server required but not found!" ) - LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_SERVER_FOUND ) - SET( XROOTD_FOUND FALSE ) -ENDIF() - ################################################################################ # XRootD posix libs # - libXrdPosix # - libXrdPosixPreload ################################################################################ FIND_LIBRARY( XROOTD_POSIX_LIBRARY XrdPosix - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd - PATH_SUFFIXES lib lib64 + PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) FIND_LIBRARY( XROOTD_POSIX_PRELOAD_LIBRARY XrdPosixPreload - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd - PATH_SUFFIXES lib lib64 + PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_POSIX_LIBRARY}" STREQUAL "XROOTD_POSIX_LIBRARY-NOTFOUND" ) @@ -155,23 +116,12 @@ IF( NOT "${XROOTD_POSIX_LIBRARY}" STREQUAL "XROOTD_POSIX_LIBRARY-NOTFOUND" ) ENDIF() ENDIF() -IF( XRootD_FIND_REQUIRED_POSIX AND NOT XROOTD_POSIX_FOUND ) - MESSAGE( "XRootD posix required but not found!" ) - LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_POSIX_FOUND ) - SET( XROOTD_FOUND FALSE ) -ENDIF() - ################################################################################ # XRootD HTTP (XrdHttp) libs # - libXrdHtppUtils ################################################################################ FIND_LIBRARY( XROOTD_HTTP_LIBRARIES XrdHttpUtils - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd - PATH_SUFFIXES lib lib64 + PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_HTTP_LIBRARIES}" STREQUAL "XROOTD_HTTP_LIBRARIES-NOTFOUND" ) @@ -179,33 +129,17 @@ IF( NOT "${XROOTD_HTTP_LIBRARIES}" STREQUAL "XROOTD_HTTP_LIBRARIES-NOTFOUND" ) LIST( APPEND XROOTD_LIBRARIES ${XROOTD_HTTP_LIBRARIES} ) ENDIF() -IF( XRootD_FIND_REQUIRED_HTTP AND NOT XROOTD_HTTP_FOUND ) - MESSAGE( "XRootD http required but not found!" ) - LIST( APPEND _XROOTD_MISSING_COMPONENTS XROOTD_HTTP_FOUND ) - SET( XROOTD_FOUND FALSE ) -ENDIF() - ################################################################################ # XRootD SSI libs # - XrdSsiLib # - XrdSsiShMap ################################################################################ FIND_LIBRARY( XROOTD_SSI_LIBRARY XrdSsiLib - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd - PATH_SUFFIXES lib lib64 + PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) FIND_LIBRARY( XROOTD_SSI_SHMAP_LIBRARY XrdSsiShMap - HINTS - ${XROOTD_DIR} - $ENV{XROOTD_DIR} - /usr - /opt/xrootd - PATH_SUFFIXES lib lib64 + PATHS ${XRootD_LIB_DIR} NO_DEFAULT_PATH ) IF( NOT "${XROOTD_SSI_LIBRARY}" STREQUAL "XROOTD_SSI_LIBRARY-NOTFOUND" ) @@ -216,27 +150,42 @@ IF( NOT "${XROOTD_SSI_LIBRARY}" STREQUAL "XROOTD_SSI_LIBRARY-NOTFOUND" ) ENDIF() ENDIF() -IF( XRootD_FIND_REQUIRED_SSI AND NOT XROOTD_SSI_FOUND ) - MESSAGE( "XRootD ssi required but not found!" ) - LIST (APPEND _XROOTD_MISSING_COMPONENTS XROOTD_SSI_FOUND ) - SET( XROOTD_FOUND FALSE ) -ENDIF() - -################################################################################ -# Utility variables for plug-in development -################################################################################ -IF( XROOTD_FOUND ) - SET( XROOTD_PLUGIN_VERSION @PLUGIN_VERSION@ ) -ENDIF() - ################################################################################ # Set up the XRootD find module ################################################################################ -IF( XRootD_FIND_REQUIRED ) - INCLUDE( FindPackageHandleStandardArgs ) - FIND_PACKAGE_HANDLE_STANDARD_ARGS( XRootD - REQUIRED_VARS XROOTD_INCLUDE_DIRS ${_XROOTD_MISSING_COMPONENTS} - ) -ENDIF() +foreach(COMPONENT UTILS CLIENT SERVER HTTP POSIX SSI) + # Set uppercase names to keep backward compatibility + set(XRootD_${COMPONENT}_FOUND ${XROOTD_${COMPONENT}_FOUND}) + set(XRootD_${COMPONENT}_LIBRARIES ${XROOTD_${COMPONENT}_LIBRARIES}) +endforeach() + +check_required_components(XRootD) + +set(XROOTD_FOUND ${XRootD_FOUND}) +set(XRootD_LIBRARIES ${XROOTD_LIBRARIES}) + +message(DEBUG "XRootD_VERSION = '${XRootD_VERSION}'") +message(DEBUG "XRootD_VERSION_MAJOR = '${XRootD_VERSION_MAJOR}'") +message(DEBUG "XRootD_VERSION_MINOR = '${XRootD_VERSION_MINOR}'") +message(DEBUG "XRootD_VERSION_PATCH = '${XRootD_VERSION_PATCH}'") +message(TRACE "XRootD_VERSION_TWEAK = '${XRootD_VERSION_TWEAK}'") +message(TRACE "XRootD_VERSION_NUMBER = '${XRootD_VERSION_NUMBER}'") + +message(TRACE "XRootD_FOUND = '${XRootD_FOUND}'") +message(TRACE "XRootD_INSTALL_PREFIX = '@CMAKE_INSTALL_PREFIX@'") +message(TRACE "XRootD_INCLUDE_DIR = '${XRootD_INCLUDE_DIR}'") +message(TRACE "XRootD_LIBRARY_DIR = '${XRootD_LIBRARY_DIR}'") +message(TRACE "XRootD_LIBRARIES = '${XRootD_LIBRARIES}'") + +foreach(COMPONENT UTILS CLIENT SERVER HTTP POSIX SSI) + message(TRACE "XRootD_${COMPONENT}_FOUND\t\t= '${XRootD_${COMPONENT}_FOUND}'") +endforeach() + +foreach(COMPONENT UTILS CLIENT SERVER HTTP POSIX SSI) + message(TRACE "XRootD_${COMPONENT}_LIBRARIES\t= '${XRootD_${COMPONENT}_LIBRARIES}'") +endforeach() +include(FindPackageHandleStandardArgs) +set(XRootD_CONFIG ${CMAKE_CURRENT_LIST_FILE}) +find_package_handle_standard_args(XRootD CONFIG_MODE HANDLE_COMPONENTS) diff --git a/cmake/XRootDDefaults.cmake b/cmake/XRootDDefaults.cmake index f9bccf6549d..f7223ceadd2 100644 --- a/cmake/XRootDDefaults.cmake +++ b/cmake/XRootDDefaults.cmake @@ -19,8 +19,6 @@ option( ENABLE_XRDCL "Enable XRootD client." option( ENABLE_TESTS "Enable unit tests." FALSE ) option( ENABLE_HTTP "Enable HTTP component." TRUE ) option( ENABLE_PYTHON "Enable python bindings." TRUE ) -# As PIP_OPTIONS uses the cache, make sure to clean cache if rebuilding (e.g. cmake --build --clean-first) -SET(PIP_OPTIONS "" CACHE STRING "pip options used during the Python bindings install.") option( XRDCL_ONLY "Build only the client and necessary dependencies" FALSE ) option( XRDCL_LIB_ONLY "Build only the client libraries and necessary dependencies" FALSE ) option( PYPI_BUILD "The project is being built for PyPI release" FALSE ) diff --git a/cmake/XRootDFindLibs.cmake b/cmake/XRootDFindLibs.cmake index 7e98653b97f..30775cca90f 100644 --- a/cmake/XRootDFindLibs.cmake +++ b/cmake/XRootDFindLibs.cmake @@ -105,6 +105,9 @@ endif() if( ENABLE_MACAROONS ) if( FORCE_ENABLED ) + if( NOT BUILD_HTTP ) + message(SEND_ERROR "Cannot enable XrdMacaroons without HTTP support") + endif() find_package( Macaroons REQUIRED ) else() find_package( Macaroons ) @@ -113,13 +116,11 @@ if( ENABLE_MACAROONS ) include(FindPkgConfig REQUIRED) if( FORCE_ENABLED ) pkg_check_modules(JSON REQUIRED json-c) - pkg_check_modules(UUID REQUIRED uuid) else() pkg_check_modules(JSON json-c) - pkg_check_modules(UUID uuid) endif() endif() - if( MACAROONS_FOUND AND JSON_FOUND AND UUID_FOUND ) + if( MACAROONS_FOUND AND JSON_FOUND AND BUILD_HTTP ) set( BUILD_MACAROONS TRUE ) else() set( BUILD_MACAROONS FALSE ) @@ -141,31 +142,49 @@ if( ENABLE_SCITOKENS ) endif() if( ENABLE_XRDEC ) - if( FORCE_ENABLED ) - find_package( Yasm REQUIRED ) - find_package( LibTool REQUIRED ) - find_package( AutoMake REQUIRED ) - find_package( AutoConf REQUIRED ) - else() - find_package( Yasm ) - find_package( LibTool ) - find_package( AutoMake ) - find_package( AutoConf ) - endif() - if( YASM_FOUND AND LIBTOOL_FOUND AND AUTOMAKE_FOUND AND AUTOCONF_FOUND ) - set( BUILD_XRDEC TRUE ) + if( USE_SYSTEM_ISAL ) + if( FORCE_ENABLED ) + find_package(isal REQUIRED) + else() + find_package(isal) + endif() + if( ISAL_FOUND ) + set(BUILD_XRDEC TRUE) + else() + set(BUILD_XRDEC FALSE) + endif() else() - set( BUILD_XRDEC FALSE ) + if( FORCE_ENABLED ) + find_package( Yasm REQUIRED ) + find_package( LibTool REQUIRED ) + find_package( AutoMake REQUIRED ) + find_package( AutoConf REQUIRED ) + else() + find_package( Yasm ) + find_package( LibTool ) + find_package( AutoMake ) + find_package( AutoConf ) + endif() + if( YASM_FOUND AND LIBTOOL_FOUND AND AUTOMAKE_FOUND AND AUTOCONF_FOUND ) + set( BUILD_XRDEC TRUE ) + else() + set( BUILD_XRDEC FALSE ) + endif() endif() endif() -if( ENABLE_PYTHON AND (LINUX OR KFREEBSD OR Hurd OR MacOSX) ) - if( FORCE_ENABLED ) - find_package( Python ${XRD_PYTHON_REQ_VERSION} COMPONENTS Interpreter Development REQUIRED ) +if( ENABLE_PYTHON OR PYPI_BUILD ) + if( CMAKE_VERSION VERSION_LESS 3.18 ) + set(PYTHON_COMPONENTS Interpreter Development) + else() + set(PYTHON_COMPONENTS Interpreter Development.Module) + endif() + if( FORCE_ENABLED OR PYPI_BUILD ) + find_package( Python ${XRD_PYTHON_REQ_VERSION} REQUIRED COMPONENTS ${PYTHON_COMPONENTS} ) else() - find_package( Python ${XRD_PYTHON_REQ_VERSION} COMPONENTS Interpreter Development ) + find_package( Python ${XRD_PYTHON_REQ_VERSION} COMPONENTS ${PYTHON_COMPONENTS} ) endif() - if( Python_Interpreter_FOUND AND Python_Development_FOUND ) + if( Python_FOUND ) set( BUILD_PYTHON TRUE ) else() set( BUILD_PYTHON FALSE ) diff --git a/cmake/XRootDOSDefs.cmake b/cmake/XRootDOSDefs.cmake index 3d4ae4abc73..475fcfe33da 100644 --- a/cmake/XRootDOSDefs.cmake +++ b/cmake/XRootDOSDefs.cmake @@ -19,6 +19,7 @@ define_default( LIBRARY_PATH_PREFIX "lib" ) # Enable c++14 #------------------------------------------------------------------------------- set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED TRUE) if( ENABLE_ASAN ) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") @@ -107,9 +108,9 @@ if( APPLE ) set( MacOSX TRUE ) set( XrdClPipelines TRUE ) - if( NOT DEFINED CMAKE_MACOSX_RPATH ) - set( CMAKE_MACOSX_RPATH 1 ) - endif() + set(CMAKE_MACOSX_RPATH TRUE) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + set(CMAKE_INSTALL_RPATH "@loader_path/../lib") # this is here because of Apple deprecating openssl and krb5 set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-declarations" ) diff --git a/cmake/XRootDSummary.cmake b/cmake/XRootDSummary.cmake index a64fcac3e98..7df55dda4e5 100644 --- a/cmake/XRootDSummary.cmake +++ b/cmake/XRootDSummary.cmake @@ -2,20 +2,20 @@ # Print the configuration summary #------------------------------------------------------------------------------- set( TRUE_VAR TRUE ) -component_status( READLINE ENABLE_READLINE READLINE_FOUND ) +component_status( CEPH XRDCEPH_SUBMODULE TRUE_VAR) component_status( FUSE BUILD_FUSE FUSE_FOUND ) +component_status( HTTP BUILD_HTTP OPENSSL_FOUND ) component_status( KRB5 BUILD_KRB5 KERBEROS5_FOUND ) -component_status( XRDCL ENABLE_XRDCL TRUE_VAR ) -component_status( XRDCLHTTP ENABLE_XRDCLHTTP DAVIX_FOUND ) +component_status( MACAROONS BUILD_MACAROONS MACAROONS_FOUND AND JSON_FOUND AND BUILD_HTTP ) +component_status( PYTHON BUILD_PYTHON Python_Interpreter_FOUND AND Python_Development_FOUND ) +component_status( READLINE ENABLE_READLINE READLINE_FOUND ) +component_status( SCITOKENS BUILD_SCITOKENS SCITOKENSCPP_FOUND ) component_status( TESTS BUILD_TESTS CPPUNIT_FOUND AND GTEST_FOUND ) -component_status( HTTP BUILD_HTTP OPENSSL_FOUND ) component_status( TPC BUILD_TPC CURL_FOUND ) -component_status( MACAROONS BUILD_MACAROONS MACAROONS_FOUND ) -component_status( PYTHON BUILD_PYTHON Python_Interpreter_FOUND AND Python_Development_FOUND ) component_status( VOMSXRD BUILD_VOMS VOMS_FOUND ) -component_status( XRDEC ENABLE_XRDEC TRUE_VAR ) -component_status( MACAROONS BUILD_MACAROONS MACAROONS_FOUND AND JSON_FOUND AND UUID_FOUND ) -component_status( SCITOKENS BUILD_SCITOKENS SCITOKENSCPP_FOUND ) +component_status( XRDCL ENABLE_XRDCL TRUE_VAR ) +component_status( XRDCLHTTP ENABLE_XRDCLHTTP DAVIX_FOUND ) +component_status( XRDEC BUILD_XRDEC TRUE_VAR ) message( STATUS "----------------------------------------" ) message( STATUS "Installation path: " ${CMAKE_INSTALL_PREFIX} ) @@ -24,19 +24,26 @@ message( STATUS "C++ Compiler: " ${CMAKE_CXX_COMPILER} ) message( STATUS "Build type: " ${CMAKE_BUILD_TYPE} ) message( STATUS "Plug-in version: " ${PLUGIN_VERSION} ) message( STATUS "" ) +message( STATUS "Ceph support: " ${STATUS_CEPH} ) message( STATUS "Readline support: " ${STATUS_READLINE} ) -message( STATUS "Fuse support: " ${STATUS_FUSE} ) -message( STATUS "Crypto support: " ${STATUS_CRYPTO} ) +message( STATUS "FUSE support: " ${STATUS_FUSE} ) message( STATUS "Kerberos5 support: " ${STATUS_KRB5} ) message( STATUS "XrdCl: " ${STATUS_XRDCL} ) message( STATUS "XrdClHttp: " ${STATUS_XRDCLHTTP} ) -message( STATUS "Tests: " ${STATUS_TESTS} ) message( STATUS "HTTP support: " ${STATUS_HTTP} ) message( STATUS "HTTP TPC support: " ${STATUS_TPC} ) -message( STATUS "Macaroons support: " ${STATUS_MACAROONS} ) message( STATUS "VOMS support: " ${STATUS_VOMSXRD} ) message( STATUS "Python support: " ${STATUS_PYTHON} ) -message( STATUS "XrdEc: " ${STATUS_XRDEC} ) +message( STATUS "Erasure coding: " ${STATUS_XRDEC} ) message( STATUS "Macaroons: " ${STATUS_MACAROONS} ) message( STATUS "SciTokens: " ${STATUS_SCITOKENS} ) +message( STATUS "Tests: " ${STATUS_TESTS} ) message( STATUS "----------------------------------------" ) + +if( FORCE_ENABLED ) + foreach(FEATURE CEPH FUSE HTTP KRB5 MACAROONS PYTHON READLINE SCITOKENS TESTS VOMSXRD XRDCL XRDCLHTTP) + if(ENABLE_${FEATURE} AND NOT STATUS_${FEATURE} STREQUAL "yes") + message(SEND_ERROR "Could not enable feature: ${FEATURE}") + endif() + endforeach() +endif() diff --git a/cmake/XRootDVersion.cmake b/cmake/XRootDVersion.cmake new file mode 100644 index 00000000000..8435a05f1e5 --- /dev/null +++ b/cmake/XRootDVersion.cmake @@ -0,0 +1,77 @@ +#.rst: +# +# XRootDVersion +# ------------- +# +# This module sets the version of XRootD. +# +# The version is determined in the following order: +# * If a version is set with -DXRootD_VERSION_STRING=x.y.z during configuration, it is used. +# * The version is read from the 'VERSION' file at the top directory of the repository. +# * If the 'VERSION' file has not been expanded, a version is set using git describe. +# * If none of the above worked, a fallback version is set using the current date. +# + +if(NOT DEFINED XRootD_VERSION_STRING) + file(READ "${PROJECT_SOURCE_DIR}/VERSION" XRootD_VERSION_STRING) + string(STRIP ${XRootD_VERSION_STRING} XRootD_VERSION_STRING) +endif() + +if(XRootD_VERSION_STRING MATCHES "Format:" AND IS_DIRECTORY ${PROJECT_SOURCE_DIR}/.git) + find_package(Git QUIET) + if(Git_FOUND) + message(VERBOSE "Determining version with git") + execute_process(COMMAND + ${GIT_EXECUTABLE} --git-dir ${PROJECT_SOURCE_DIR}/.git describe + OUTPUT_VARIABLE XRootD_VERSION_STRING ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + set(XRootD_VERSION_STRING "${XRootD_VERSION_STRING}" CACHE INTERNAL "XRootD Version") + endif() +endif() + +if(XRootD_VERSION_STRING MATCHES "^v?([0-9]+)[.]*([0-9]*)[.]*([0-9]*)[.]*([0-9]*)") + set(XRootD_VERSION_MAJOR ${CMAKE_MATCH_1}) + if(${CMAKE_MATCH_COUNT} GREATER 1) + set(XRootD_VERSION_MINOR ${CMAKE_MATCH_2}) + else() + set(XRootD_VERSION_MINOR 0) + endif() + if(${CMAKE_MATCH_COUNT} GREATER 2) + set(XRootD_VERSION_PATCH ${CMAKE_MATCH_3}) + else() + set(XRootD_VERSION_PATCH 0) + endif() + if(${CMAKE_MATCH_COUNT} GREATER 3) + set(XRootD_VERSION_TWEAK ${CMAKE_MATCH_4}) + else() + set(XRootD_VERSION_TWEAK 0) + endif() + math(EXPR XRootD_VERSION_NUMBER + "10000 * ${XRootD_VERSION_MAJOR} + 100 * ${XRootD_VERSION_MINOR} + ${XRootD_VERSION_PATCH}" + OUTPUT_FORMAT DECIMAL) +else() + message(WARNING "Failed to determine XRootD version, using a timestamp as fallback." + "You can override this by setting -DXRootD_VERSION_STRING=x.y.z during configuration.") + set(XRootD_VERSION_MAJOR 5) + set(XRootD_VERSION_MINOR 7) + set(XRootD_VERSION_PATCH 0) + set(XRootD_VERSION_TWEAK 0) + set(XRootD_VERSION_NUMBER 1000000) + string(TIMESTAMP XRootD_VERSION_STRING + "v${XRootD_VERSION_MAJOR}.${XRootD_VERSION_MINOR}-rc%Y%m%d" UTC) +endif() + +if(XRootD_VERSION_STRING MATCHES "[_-](.*)$") + set(XRootD_VERSION_SUFFIX ${CMAKE_MATCH_1}) +endif() + +string(REGEX MATCH "[0-9]+[.]*[0-9]*[.]*[0-9]*[.]*[0-9]*(-rc)?[0-9].*" + XRootD_VERSION ${XRootD_VERSION_STRING}) + +message(DEBUG "XRootD_VERSION_STRING = '${XRootD_VERSION_STRING}'") +message(DEBUG "XRootD_VERSION_NUMBER = '${XRootD_VERSION_NUMBER}'") +message(DEBUG "XRootD_VERSION_MAJOR = '${XRootD_VERSION_MAJOR}'") +message(DEBUG "XRootD_VERSION_MINOR = '${XRootD_VERSION_MINOR}'") +message(DEBUG "XRootD_VERSION_PATCH = '${XRootD_VERSION_PATCH}'") +message(DEBUG "XRootD_VERSION_TWEAK = '${XRootD_VERSION_TWEAK}'") +message(DEBUG "XRootD_VERSION_SUFFIX = '${XRootD_VERSION_SUFFIX}'") +message(DEBUG "XRootD_VERSION = '${XRootD_VERSION}'") diff --git a/debian/compat b/debian/compat new file mode 100644 index 00000000000..b1bd38b62a0 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +13 diff --git a/debian/control b/debian/control new file mode 100644 index 00000000000..f447fe51b9c --- /dev/null +++ b/debian/control @@ -0,0 +1,408 @@ +Source: xrootd +Section: net +Priority: optional +Standards-Version: 4.6.2 +Build-Depends: + debhelper (>= 13), + dh-python, + attr, + cmake, + libgtest-dev, + libcppunit-dev, + libisal-dev, + pkg-config, + libfuse-dev [linux-any kfreebsd-any], + libkrb5-dev, + libcurl4-openssl-dev, + libtinyxml-dev, + libxml2-dev, + ncurses-dev, + libssl-dev, + libreadline-dev, + zlib1g-dev, + libsystemd-dev [linux-any], + python3-dev, + python3-pip, + python3-wheel, + libjson-c-dev, + libmacaroons-dev, + uuid-dev, + uuid-runtime, + voms-dev, + libscitokens-dev, + davix-dev, + librados-dev [i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x], + libradospp-dev [i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x], + libradosstriper-dev [i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x] +Build-Depends-Indep: + dh-sequence-sphinxdoc, + doxygen, + graphviz, + python3-sphinx +Homepage: http://xrootd.org/ +Maintainer: XRootD Developers +Vcs-Browser: https://github.com/xrootd/xrootd +Vcs-Git: https://github.com/xrootd/xrootd.git + +Package: xrootd-server +Architecture: any +Multi-Arch: foreign +Section: net +Depends: + xrootd-plugins (= ${binary:Version}), + xrootd-server-plugins (= ${binary:Version}), + expect, + logrotate, + adduser, + ${perl:Depends}, + ${shlibs:Depends}, + ${misc:Depends} +Description: Extended ROOT file server + The Extended root file server consists of a file server called xrootd + and a cluster management server called cmsd. + . + The xrootd server was developed for the root analysis framework to + serve root files. However, the server is agnostic to file types and + provides POSIX-like access to any type of file. + . + The cmsd server is the next generation version of the olbd server, + originally developed to cluster and load balance Objectivity/DB AMS + database servers. It provides enhanced capability along with lower + latency and increased throughput. + +Package: libxrdapputils2 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Utilities library for xrootd applications + This package contains the xrootd utilities library for applications. + +Package: libxrdcrypto2 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Cryptograpic library for xrootd + This package contains the xrootd cryptograpic library. + +Package: libxrdcryptolite2 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Light version of cryptograpic library for xrootd + This package contains the light version of the xrootd cryptograpic library. + +Package: libxrdutils3 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Utilities library for xrootd + This package contains the xrootd utilities library. + +Package: libxrdxml3 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: XML library for xrootd + This package contains the xrootd XML library. + +Package: xrootd-plugins +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Plugins used by xrootd servers and clients + This package contains plugins used by the xrootd servers and clients. + +Package: libxrootd-dev +Architecture: any +Multi-Arch: same +Section: libdevel +Depends: + libxrdapputils2 (= ${binary:Version}), + libxrdcrypto2 (= ${binary:Version}), + libxrdcryptolite2 (= ${binary:Version}), + libxrdutils3 (= ${binary:Version}), + libxrdxml3 (= ${binary:Version}), + ${misc:Depends} +Description: Development files for xrootd + This package contains header files and development libraries for xrootd + development. + +Package: libxrdcl3 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Recommends: + xrootd-plugins (= ${binary:Version}), + xrootd-client-plugins(= ${binary:Version}) +Description: Client library for xrootd + This package contains the xrootd client library. + +Package: libxrdec1 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Recommends: + xrootd-plugins (= ${binary:Version}), + xrootd-client-plugins(= ${binary:Version}) +Description: Client library for xrootd + This package contains the xrootd client library for erasure coding. + +Package: libxrdffs3 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: File protocol library for xrootd + This package contains the xrootd file protocol library. + +Package: libxrdposix3 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Posix interface library for xrootd + This package contains the xrootd Posix interface library. + +Package: libxrdssilib2 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Server internals library for xrootd + This package contains an xrootd server internals library. + +Package: libxrdssishmap2 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Server internals library for xrootd + This package contains an xrootd server internals library. + +Package: xrootd-client-plugins +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Plugins used by xrootd clients + This package contains plugins used by xrootd clients. + +Package: xrootd-client-http-plugins +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: HTTP client plugin for XRootD client + This package contains an XRootD client plugin which allows XRootD to + interact with HTTP repositories. + +Package: libxrootd-client-dev +Architecture: any +Multi-Arch: same +Section: libdevel +Depends: + libxrdcl3 (= ${binary:Version}), + libxrdffs3 (= ${binary:Version}), + libxrdposix3 (= ${binary:Version}), + libxrootd-dev (= ${binary:Version}), + ${misc:Depends} +Description: Development files for xrootd clients + This package contains header files and development libraries for xrootd + client development. + +Package: libxrdhttputils2 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: HTTP protocol utilities library for xrootd + This package contains the xrootd HTTP protocol utilities library. + +Package: libxrdserver3 +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Recommends: + xrootd-plugins (= ${binary:Version}), + xrootd-server-plugins(= ${binary:Version}) +Description: Server library for xrootd + This package contains the xrootd server library. + +Package: xrootd-server-plugins +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Plugins used by xrootd servers + This package contains plugins used by xrootd servers. + +Package: libxrootd-server-dev +Architecture: any +Multi-Arch: same +Section: libdevel +Depends: + libxrdhttputils2 (= ${binary:Version}), + libxrdserver3 (= ${binary:Version}), + libxrootd-dev (= ${binary:Version}), + libxrootd-client-dev (= ${binary:Version}), + ${misc:Depends} +Description: Development files for xrootd servers + This package contains header files and development libraries for xrootd + server development. + +Package: libxrootd-private-dev +Architecture: any +Multi-Arch: same +Section: libdevel +Depends: + libxrdssilib2 (= ${binary:Version}), + libxrdssishmap2 (= ${binary:Version}), + libxrootd-dev (= ${binary:Version}), + libxrootd-client-dev (= ${binary:Version}), + libxrootd-server-dev (= ${binary:Version}), + ${misc:Depends} +Description: Private xrootd headers + This package contains some private xrootd headers. Backward and forward + compatibility between versions is not guaranteed for these headers. + +Package: xrootd-client +Architecture: any +Multi-Arch: foreign +Section: net +Depends: + xrootd-plugins (= ${binary:Version}), + xrootd-client-plugins(= ${binary:Version}), + ${shlibs:Depends}, + ${misc:Depends} +Description: Xrootd command line client tools + This package contains the command line tools used to communicate with + xrootd servers. + +Package: xrootd-fuse +Architecture: linux-any kfreebsd-any +Multi-Arch: foreign +Section: net +Depends: + xrootd-plugins (= ${binary:Version}), + xrootd-client-plugins (= ${binary:Version}), + fuse, + ${shlibs:Depends}, + ${misc:Depends} +Description: Xrootd FUSE tool + This package contains the FUSE (file system in user space) xrootd mount + tool. + +Package: xrootd-voms-plugins +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: VOMS attribute extractor plugin for XRootD + This package contains the xrootd VOMS attribute extractor plugin. + +Package: xrootd-scitokens-plugins +Architecture: any +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: SciTokens authorization support for XRootD + This ACC (authorization) plugin for the XRootD framework utilizes the + SciTokens library to validate and extract authorization claims from a + SciToken passed during a transfer. Configured appropriately, this + allows the XRootD server admin to delegate authorization decisions for + a subset of the namespace to an external issuer. + +Package: libxrdcephposix0 +Architecture: i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: Ceph posix library for xrootd + This package contains an xrootd library used by the ceph plugins. + +Package: xrootd-ceph-plugins +Architecture: i386 amd64 armel armhf arm64 mips mipsel mips64el powerpc ppc64el riscv64 s390x +Multi-Arch: same +Section: libs +Depends: + ${shlibs:Depends}, + ${misc:Depends} +Description: XRootD plugin for interfacing with the Ceph storage platform + The xrootd-ceph is an OSS layer plugin for the XRootD server for + interfacing with the Ceph storage platform. + +Package: python3-xrootd +Architecture: any +Multi-Arch: foreign +Section: python +Provides: + ${python3:Provides} +Depends: + xrootd-plugins (= ${binary:Version}), + xrootd-client-plugins (= ${binary:Version}), + ${python3:Depends}, + ${shlibs:Depends}, + ${misc:Depends} +Description: Python 3 bindings for xrootd + This package contains Python 3 bindings for xrootd. + +Package: xrootd-doc +Architecture: all +Multi-Arch: foreign +Section: doc +Depends: + ${sphinxdoc:Depends}, + ${misc:Depends} +Built-Using: + ${sphinxdoc:Built-Using} +Description: Developer documentation for the xrootd libraries + This package contains the API documentation of the xrootd libraries. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 00000000000..c0c64a30374 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,320 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: xrootd +Upstream-Contact: https://xrootd.slac.stanford.edu/ + +Files: * +Copyright: + 2000-2023 Board of Trustees of the Leland Stanford, Jr. University. + Produced under contract DE-AC02-76-SF00515 with the US Department of + Energy. All rights reserved. +License: LGPL-3 and BSD-3-clause-modified + The copyright holder's institutional names may not be used to endorse + or promote products derived from this software without specific prior + written permission. + . + This file is part of the XRootD software suite. + . + XRootD is free software: you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + XRootD is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + . + You should have received a copy of the GNU Lesser General Public + License along with XRootD in a file called COPYING.LESSER (LGPL + license) and file COPYING (GPL license). If not, see + . + . + On Debian systems the full text of the GNU lesser general public + license version 3 can be found in /usr/share/common-licenses/LGPL-3. + . + Prior to September 2nd, 2012 the XRootD software suite was licensed + under a modified BSD license shown below. This applies to all code + that was in the XRootD git repository prior to that date. All code is + now licensed under LGPL. + . + Conditions of Use + . + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + . + a. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + b. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + c. Neither the name of the Leland Stanford, Jr. University nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + d. Products derived from this software that do not adhere to the + xrootd or cmsd protocol specifications may not use the acronyms + 'cmsd', 'Scalla', 'xroot', and 'xrootd', regardless of + capitalization, to describe such derivative works. + . + DISCLAIMER + . + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + 'AS IS' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Files: + bindings/python/* + src/XrdApps/XrdClProxyPlugin/* + src/XrdApps/XrdClRecordPlugin/* + src/XrdCeph/* + src/XrdCks/XrdCksCalczcrc32.cc + src/XrdCl/* + src/XrdEc/* + src/XrdHttp/* + src/XrdOssCsi/* + src/XrdOuc/XrdOucCompiler.hh + src/XrdOuc/XrdOucEnum.hh + src/XrdSys/XrdSysKernelBuffer.hh + src/XrdTls/XrdTlsContext.* + src/XrdTls/XrdTlsSocket.* + src/XrdZip/* + tests/common/* + tests/XrdCephTests/* + tests/XrdClTests/* + tests/XrdEcTests/* + utils/xrootd-config +Copyright: + 2011-2023 European Organization for Nuclear Research (CERN) +Comment: + In applying this licence, CERN does not waive the privileges and immunities + granted to it by virtue of its status as an Intergovernmental Organization + or submit itself to any jurisdiction. +License: LGPL-3 + +Files: + src/XrdCks/XrdCksCalcadler32.hh + src/XrdOuc/XrdOucCRC32C.* +Copyright: + 1995-1998, 2013, 2015 Mark Adler +License: zlib/libpng + +Files: + src/XrdCl/XrdClLocalFileHandler.* + src/XrdCl/XrdClLocalFileTask.* + src/XrdCms/XrdCmsRedirLocal.* + tests/XrdClTests/LocalFileHandlerTest.cc +Copyright: + 2017, 2019 GSI Helmholtzzentrum für Schwerionenforschung GmbH +License: LGPL-3 + +Files: src/XrdCl/XrdClMonitor.hh +Copyright: + 2012 Board of Trustees of the Leland Stanford, Jr., University + 2012 European Organization for Nuclear Research (CERN) +License: LGPL-3 + +Files: src/XrdOuc/XrdOucJson.hh +Copyright: + 2013-2019 Niels Lohmann +License: Expat + +Files: src/XrdOuc/XrdOucSHA3.* +Copyright: + 2015 Markku-Juhani O. Saarinen +License: Expat + +Files: src/XrdOuc/XrdOucUri.* +Copyright: + 2016 by David Farrell +License: BSD-2-clause + +Files: src/XrdSciTokens/* +Copyright: + Brian Bockelman, Andrew Hanushevsky, Derek Weitzel +License: Apache-2.0 + On Debian systems the full text of the Apache license version 2 can + be found in /usr/share/common-licenses/Apache-2.0. + +Files: src/XrdSciTokens/vendor/inih/* +Copyright: + 2009, Ben Hoyt. All rights reserved. +License: BSD-3-clause + +Files: src/XrdSciTokens/vendor/picojson/* +Copyright: + 2009-2010 Cybozu Labs, Inc. + 2011-2014 Kazuho Oku + All rights reserved. +License: BSD-2-clause + +Files: src/XrdSciTokens/vendor/picojson/picotest/* +Copyright: + 2014 DeNA Co., Ltd. +License: Expat + +Files: src/XrdSecztn/XrdSecztn.cc +Copyright: + 2015 Erwin Jansen +License: Expat + +Files: src/XrdSys/XrdSysFallocate.* +Copyright: + 2010 Mozilla Foundation. All Rights Reserved. + 2015, 2016 R.J.V. Bertin for KDE project. +Comment: + Original license allows modification / redistribution under LGPL 2.1 + or higher. Redistributed under LGPL-3 or higher. +License: LGPL-3 + +Files: src/XrdTls/XrdTlsHostcheck.* +Copyright: + 1998-2012 Daniel Stenberg, , et al. +License: curl + This software is licensed as described in the file COPYING, which + you should have received as part of this distribution. The terms + are also available at http://curl.haxx.se/docs/copyright.html. + . + You may opt to use, copy, modify, merge, publish, distribute and/or sell + copies of the Software, and permit persons to whom the Software is + furnished to do so, under the terms of the COPYING file. + . + This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + KIND, either express or implied. + +Files: src/XrdTls/XrdTlsNotaryUtils.* +Copyright: + 2012 iSEC Partners. +License: Expat + +Files: src/XrdXml/tinyxml/* +Copyright: + 2000-2006 Lee Thomason (www.grinninglizard.com) +Comment: Not used in debian build, libtinyxml-dev used instead. +License: zlib/libpng + +Files: debian/* +Copyright: + 2020-2023 Mattias Ellert +License: LGPL-3 + +License: LGPL-3 + This file is part of the XRootD software suite. + . + XRootD is free software: you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + XRootD is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + . + You should have received a copy of the GNU Lesser General Public + License along with XRootD in a file called COPYING.LESSER (LGPL + license) and file COPYING (GPL license). If not, see + . + . + On Debian systems the full text of the GNU lesser general public + license version 3 can be found in /usr/share/common-licenses/LGPL-3. + +License: zlib/libpng + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any + damages arising from the use of this software. + . + Permission is granted to anyone to use this software for any + purpose, including commercial applications, and to alter it and + redistribute it freely, subject to the following restrictions: + . + 1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + . + 2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + . + 3. This notice may not be removed or altered from any source + distribution. + +License: BSD-2-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + . + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + . + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License: BSD-3-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + . + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of [insert "CERN" or "Ben Hoyt" as appropriate] + nor the names of its contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + . + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License: Expat + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. diff --git a/packaging/debian/libxrdapputils1.install b/debian/libxrdapputils1.install similarity index 100% rename from packaging/debian/libxrdapputils1.install rename to debian/libxrdapputils1.install diff --git a/debian/libxrdapputils2.install b/debian/libxrdapputils2.install new file mode 100644 index 00000000000..93362c3e45c --- /dev/null +++ b/debian/libxrdapputils2.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdAppUtils.so.* diff --git a/debian/libxrdcephposix0.install b/debian/libxrdcephposix0.install new file mode 100644 index 00000000000..36bd6c377bd --- /dev/null +++ b/debian/libxrdcephposix0.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdCephPosix.so.* diff --git a/packaging/debian/libxrdcl2.install b/debian/libxrdcl2.install similarity index 100% rename from packaging/debian/libxrdcl2.install rename to debian/libxrdcl2.install diff --git a/debian/libxrdcl3.install b/debian/libxrdcl3.install new file mode 100644 index 00000000000..655caa87025 --- /dev/null +++ b/debian/libxrdcl3.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdCl.so.* diff --git a/packaging/debian/libxrdcrypto1.install b/debian/libxrdcrypto1.install similarity index 100% rename from packaging/debian/libxrdcrypto1.install rename to debian/libxrdcrypto1.install diff --git a/debian/libxrdcrypto2.install b/debian/libxrdcrypto2.install new file mode 100644 index 00000000000..46dc28cdc0a --- /dev/null +++ b/debian/libxrdcrypto2.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdCrypto.so.* diff --git a/packaging/debian/libxrdcryptolite1.install b/debian/libxrdcryptolite1.install similarity index 100% rename from packaging/debian/libxrdcryptolite1.install rename to debian/libxrdcryptolite1.install diff --git a/debian/libxrdcryptolite2.install b/debian/libxrdcryptolite2.install new file mode 100644 index 00000000000..cdb6bf81dbe --- /dev/null +++ b/debian/libxrdcryptolite2.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdCryptoLite.so.* diff --git a/debian/libxrdec1.install b/debian/libxrdec1.install new file mode 100644 index 00000000000..fe56c742a69 --- /dev/null +++ b/debian/libxrdec1.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdEc.so.* diff --git a/packaging/debian/libxrdffs2.install b/debian/libxrdffs2.install similarity index 100% rename from packaging/debian/libxrdffs2.install rename to debian/libxrdffs2.install diff --git a/debian/libxrdffs3.install b/debian/libxrdffs3.install new file mode 100644 index 00000000000..11aa0046d49 --- /dev/null +++ b/debian/libxrdffs3.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdFfs.so.* diff --git a/packaging/debian/libxrdhttputils1.install b/debian/libxrdhttputils1.install similarity index 100% rename from packaging/debian/libxrdhttputils1.install rename to debian/libxrdhttputils1.install diff --git a/debian/libxrdhttputils2.install b/debian/libxrdhttputils2.install new file mode 100644 index 00000000000..b5becae8f13 --- /dev/null +++ b/debian/libxrdhttputils2.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdHttpUtils.so.* diff --git a/packaging/debian/libxrdposix2.install b/debian/libxrdposix2.install similarity index 100% rename from packaging/debian/libxrdposix2.install rename to debian/libxrdposix2.install diff --git a/debian/libxrdposix3.install b/debian/libxrdposix3.install new file mode 100644 index 00000000000..d4b46f09cc7 --- /dev/null +++ b/debian/libxrdposix3.install @@ -0,0 +1,4 @@ +/usr/lib/*/libXrdPosix.so.* +/usr/lib/*/libXrdPosixPreload.so.* +# This lib may be used for LD_PRELOAD so the .so link needs to be included +/usr/lib/*/libXrdPosixPreload.so diff --git a/debian/libxrdposix3.lintian-overrides b/debian/libxrdposix3.lintian-overrides new file mode 100644 index 00000000000..7f1d3a729a1 --- /dev/null +++ b/debian/libxrdposix3.lintian-overrides @@ -0,0 +1,3 @@ +# This lib may be used for LD_PRELOAD so the .so link needs to be included +link-to-shared-library-in-wrong-package * [usr/lib/*/libXrdPosixPreload.so] +lacks-unversioned-link-to-shared-library * [usr/lib/*/libXrdPosixPreload.so.*] diff --git a/packaging/debian/libxrdserver2.install b/debian/libxrdserver2.install similarity index 100% rename from packaging/debian/libxrdserver2.install rename to debian/libxrdserver2.install diff --git a/debian/libxrdserver3.install b/debian/libxrdserver3.install new file mode 100644 index 00000000000..fb9075c5f83 --- /dev/null +++ b/debian/libxrdserver3.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdServer.so.* diff --git a/packaging/debian/libxrdssilib1.install b/debian/libxrdssilib1.install similarity index 100% rename from packaging/debian/libxrdssilib1.install rename to debian/libxrdssilib1.install diff --git a/debian/libxrdssilib2.install b/debian/libxrdssilib2.install new file mode 100644 index 00000000000..369c6f12382 --- /dev/null +++ b/debian/libxrdssilib2.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdSsiLib.so.* diff --git a/packaging/debian/libxrdssishmap1.install b/debian/libxrdssishmap1.install similarity index 100% rename from packaging/debian/libxrdssishmap1.install rename to debian/libxrdssishmap1.install diff --git a/debian/libxrdssishmap2.install b/debian/libxrdssishmap2.install new file mode 100644 index 00000000000..44577ea477a --- /dev/null +++ b/debian/libxrdssishmap2.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdSsiShMap.so.* diff --git a/packaging/debian/libxrdutils2.install b/debian/libxrdutils2.install similarity index 100% rename from packaging/debian/libxrdutils2.install rename to debian/libxrdutils2.install diff --git a/debian/libxrdutils3.install b/debian/libxrdutils3.install new file mode 100644 index 00000000000..8cbb616f877 --- /dev/null +++ b/debian/libxrdutils3.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdUtils.so.* diff --git a/packaging/debian/libxrdxml2.install b/debian/libxrdxml2.install similarity index 100% rename from packaging/debian/libxrdxml2.install rename to debian/libxrdxml2.install diff --git a/debian/libxrdxml3.install b/debian/libxrdxml3.install new file mode 100644 index 00000000000..25ee16e9adf --- /dev/null +++ b/debian/libxrdxml3.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdXml.so.* diff --git a/debian/libxrootd-client-dev.install b/debian/libxrootd-client-dev.install new file mode 100644 index 00000000000..dbaedb281be --- /dev/null +++ b/debian/libxrootd-client-dev.install @@ -0,0 +1,6 @@ +/usr/include/xrootd/XrdCl +/usr/include/xrootd/XrdPosix +/usr/lib/*/libXrdCl.so +/usr/lib/*/libXrdEc.so +/usr/lib/*/libXrdFfs.so +/usr/lib/*/libXrdPosix.so diff --git a/debian/libxrootd-dev.install b/debian/libxrootd-dev.install new file mode 100644 index 00000000000..e8e910d6210 --- /dev/null +++ b/debian/libxrootd-dev.install @@ -0,0 +1,16 @@ +/usr/bin/xrootd-config +/usr/include/xrootd/XProtocol +/usr/include/xrootd/Xrd +/usr/include/xrootd/XrdCks +/usr/include/xrootd/XrdNet +/usr/include/xrootd/XrdOuc +/usr/include/xrootd/XrdSec +/usr/include/xrootd/XrdSys +/usr/include/xrootd/XrdXml +/usr/include/xrootd/XrdVersion.hh +/usr/lib/*/libXrdAppUtils.so +/usr/lib/*/libXrdCrypto.so +/usr/lib/*/libXrdCryptoLite.so +/usr/lib/*/libXrdUtils.so +/usr/lib/*/libXrdXml.so +/usr/lib/*/cmake/XRootD diff --git a/debian/libxrootd-private-dev.install b/debian/libxrootd-private-dev.install new file mode 100644 index 00000000000..5b5910df5db --- /dev/null +++ b/debian/libxrootd-private-dev.install @@ -0,0 +1,3 @@ +/usr/include/xrootd/private +/usr/lib/*/libXrdSsiLib.so +/usr/lib/*/libXrdSsiShMap.so diff --git a/debian/libxrootd-server-dev.install b/debian/libxrootd-server-dev.install new file mode 100644 index 00000000000..0a267b049ce --- /dev/null +++ b/debian/libxrootd-server-dev.install @@ -0,0 +1,10 @@ +/usr/include/xrootd/XrdAcc +/usr/include/xrootd/XrdCms +/usr/include/xrootd/XrdHttp +/usr/include/xrootd/XrdOfs +/usr/include/xrootd/XrdOss +/usr/include/xrootd/XrdPfc +/usr/include/xrootd/XrdSfs +/usr/include/xrootd/XrdXrootd +/usr/lib/*/libXrdHttpUtils.so +/usr/lib/*/libXrdServer.so diff --git a/debian/python3-xrootd.install b/debian/python3-xrootd.install new file mode 100644 index 00000000000..23b15747d77 --- /dev/null +++ b/debian/python3-xrootd.install @@ -0,0 +1,3 @@ +/usr/lib/python3/dist-packages/xrootd-*.*-info +/usr/lib/python3/dist-packages/pyxrootd +/usr/lib/python3/dist-packages/XRootD diff --git a/debian/rules b/debian/rules new file mode 100755 index 00000000000..479f83d1fdc --- /dev/null +++ b/debian/rules @@ -0,0 +1,111 @@ +#!/usr/bin/make -f + +export LC_ALL=C +export DH_VERBOSE=1 +export PYBUILD_NAME=xrootd +export DEB_BUILD_MAINT_OPTIONS = hardening=+all optimize=-lto + +%: + dh $@ --with python3 --buildsystem cmake + +override_dh_auto_configure: + dh_auto_configure -- \ + -DENABLE_FUSE:BOOL=1 \ + -DENABLE_HTTP:BOOL=1 \ + -DENABLE_KRB5:BOOL=1 \ + -DENABLE_MACAROONS:BOOL=1 \ + -DENABLE_PYTHON:BOOL=1 \ + -DENABLE_READLINE:BOOL=1 \ + -DENABLE_SCITOKENS:BOOL=1 \ + -DENABLE_VOMS:BOOL=1 \ + -DENABLE_XRDCLHTTP:BOOL=1 \ + -DENABLE_XRDEC:BOOL=1 \ + -DENABLE_TESTS:BOOL=1 \ + -DFORCE_ENABLED:BOOL=1 \ + -DINSTALL_PYTHON_BINDINGS:BOOL=0 \ + -DUSE_SYSTEM_ISAL:BOOL=1 \ + -DXRDCEPH_SUBMODULE:BOOL=1 + +override_dh_auto_build: + dh_auto_build + doxygen Doxyfile + +override_dh_auto_clean: + dh_auto_clean + rm -rf doxydoc + rm -rf bindings/python/docs/build + +override_dh_auto_install: + dh_auto_install + python3 -m pip install --target debian/tmp/usr/lib/python3/dist-packages \ + --no-deps --no-build-isolation --disable-pip-version-check --verbose \ + --ignore-installed --use-pep517 obj-$(DEB_HOST_MULTIARCH)/bindings/python + + rm -f debian/tmp/usr/bin/xrdshmap + rm -f debian/tmp/usr/bin/test-runner + rm -f debian/tmp/usr/lib/*/libXrd*Test* + rm -f debian/tmp/usr/lib/*/libXrdCephPosix.so + + rm -f debian/tmp/usr/lib/python3/dist-packages/xrootd-*.*-info/direct_url.json + rm -f debian/tmp/usr/lib/python3/dist-packages/xrootd-*.*-info/RECORD + [ -r debian/tmp/usr/lib/python3/dist-packages/xrootd-*.*-info/INSTALLER ] && \ + sed -i -e s/pip/dpkg/ \ + debian/tmp/usr/lib/python3/dist-packages/xrootd-*.*-info/INSTALLER + + LD_LIBRARY_PATH=$${LD_LIBRARY_PATH}:$(CURDIR)/debian/tmp/usr/lib/$(DEB_HOST_MULTIARCH) \ + PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=$(CURDIR)/debian/tmp/usr/lib/python3/dist-packages \ + make -C bindings/python/docs html && \ + mv bindings/python/docs/build/html bindings/python/docs/build/python + + # Service unit files + mkdir -p debian/tmp/lib/systemd/system + install -m 644 packaging/common/xrootd@.service debian/tmp/lib/systemd/system + install -m 644 packaging/common/xrootd@.socket debian/tmp/lib/systemd/system + install -m 644 packaging/common/xrdhttp@.socket debian/tmp/lib/systemd/system + install -m 644 packaging/common/cmsd@.service debian/tmp/lib/systemd/system + install -m 644 packaging/common/frm_xfrd@.service debian/tmp/lib/systemd/system + install -m 644 packaging/common/frm_purged@.service debian/tmp/lib/systemd/system + mkdir -p debian/tmp/usr/lib/tmpfiles.d + install -m 644 packaging/rhel/xrootd.tmpfiles debian/tmp/usr/lib/tmpfiles.d/xrootd.conf + + # Server config + mkdir -p debian/tmp/etc/xrootd + install -m 644 -p packaging/common/xrootd-clustered.cfg \ + debian/tmp/etc/xrootd/xrootd-clustered.cfg + install -m 644 -p packaging/common/xrootd-standalone.cfg \ + debian/tmp/etc/xrootd/xrootd-standalone.cfg + install -m 644 -p packaging/common/xrootd-filecache-clustered.cfg \ + debian/tmp/etc/xrootd/xrootd-filecache-clustered.cfg + install -m 644 -p packaging/common/xrootd-filecache-standalone.cfg \ + debian/tmp/etc/xrootd/xrootd-filecache-standalone.cfg + sed 's!/usr/lib64/!!' packaging/common/xrootd-http.cfg > \ + debian/tmp/etc/xrootd/xrootd-http.cfg + + # Client config + mkdir -p debian/tmp/etc/xrootd/client.plugins.d + install -m 644 -p packaging/common/client.conf \ + debian/tmp/etc/xrootd/client.conf + sed 's!/usr/lib/!!' packaging/common/client-plugin.conf.example > \ + debian/tmp/etc/xrootd/client.plugins.d/client-plugin.conf.example + sed -e 's!/usr/lib64/!!' -e 's!-5!!' packaging/common/recorder.conf > \ + debian/tmp/etc/xrootd/client.plugins.d/recorder.conf + sed 's!/usr/lib64/!!' packaging/common/http.client.conf.example > \ + debian/tmp/etc/xrootd/client.plugins.d/xrdcl-http-plugin.conf + + chmod 644 debian/tmp/usr/share/xrootd/utils/XrdCmsNotify.pm + sed 's!/usr/bin/env perl!/usr/bin/perl!' -i \ + debian/tmp/usr/share/xrootd/utils/netchk \ + debian/tmp/usr/share/xrootd/utils/XrdCmsNotify.pm \ + debian/tmp/usr/share/xrootd/utils/XrdOlbMonPerf + + sed 's!/usr/bin/env bash!/bin/bash!' -i \ + debian/tmp/usr/bin/xrootd-config + + mkdir -p debian/tmp/etc/xrootd/config.d + + mkdir -p debian/tmp/var/log/xrootd + mkdir -p debian/tmp/var/spool/xrootd + + mkdir -p debian/tmp/etc/logrotate.d + install -m 644 -p packaging/common/xrootd.logrotate \ + debian/tmp/etc/logrotate.d/xrootd diff --git a/packaging/debian/source/format b/debian/source/format similarity index 100% rename from packaging/debian/source/format rename to debian/source/format diff --git a/debian/xrootd-ceph-plugins.install b/debian/xrootd-ceph-plugins.install new file mode 100644 index 00000000000..b457dde4c48 --- /dev/null +++ b/debian/xrootd-ceph-plugins.install @@ -0,0 +1,2 @@ +/usr/lib/*/libXrdCeph-5.so +/usr/lib/*/libXrdCephXattr-5.so diff --git a/debian/xrootd-ceph-plugins.lintian-overrides b/debian/xrootd-ceph-plugins.lintian-overrides new file mode 100644 index 00000000000..754468af41c --- /dev/null +++ b/debian/xrootd-ceph-plugins.lintian-overrides @@ -0,0 +1,2 @@ +# These are plugins - no soname on purpose +sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] diff --git a/packaging/debian/xrootd-client-devel.install b/debian/xrootd-client-devel.install similarity index 100% rename from packaging/debian/xrootd-client-devel.install rename to debian/xrootd-client-devel.install diff --git a/debian/xrootd-client-http-plugins.install b/debian/xrootd-client-http-plugins.install new file mode 100644 index 00000000000..98aa6cf5769 --- /dev/null +++ b/debian/xrootd-client-http-plugins.install @@ -0,0 +1,2 @@ +/usr/lib/*/libXrdClHttp-5.so +/etc/xrootd/client.plugins.d/xrdcl-http-plugin.conf diff --git a/debian/xrootd-client-http-plugins.lintian-overrides b/debian/xrootd-client-http-plugins.lintian-overrides new file mode 100644 index 00000000000..754468af41c --- /dev/null +++ b/debian/xrootd-client-http-plugins.lintian-overrides @@ -0,0 +1,2 @@ +# These are plugins - no soname on purpose +sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] diff --git a/packaging/debian/xrootd-client-libs.install b/debian/xrootd-client-libs.install similarity index 100% rename from packaging/debian/xrootd-client-libs.install rename to debian/xrootd-client-libs.install diff --git a/debian/xrootd-client-plugins.install b/debian/xrootd-client-plugins.install new file mode 100644 index 00000000000..d0732040476 --- /dev/null +++ b/debian/xrootd-client-plugins.install @@ -0,0 +1,5 @@ +/usr/lib/*/libXrdClProxyPlugin-5.so +/usr/lib/*/libXrdClRecorder-5.so +/etc/xrootd/client.conf +/etc/xrootd/client.plugins.d/client-plugin.conf.example +/etc/xrootd/client.plugins.d/recorder.conf diff --git a/debian/xrootd-client-plugins.lintian-overrides b/debian/xrootd-client-plugins.lintian-overrides new file mode 100644 index 00000000000..754468af41c --- /dev/null +++ b/debian/xrootd-client-plugins.lintian-overrides @@ -0,0 +1,2 @@ +# These are plugins - no soname on purpose +sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] diff --git a/debian/xrootd-client.install b/debian/xrootd-client.install new file mode 100644 index 00000000000..330a27cf985 --- /dev/null +++ b/debian/xrootd-client.install @@ -0,0 +1,18 @@ +/usr/bin/xrdadler32 +/usr/bin/xrdcks +/usr/bin/xrdcopy +/usr/bin/xrdcp +/usr/bin/xrdcrc32c +/usr/bin/xrdfs +/usr/bin/xrdgsiproxy +/usr/bin/xrdgsitest +/usr/bin/xrdmapc +/usr/bin/xrdpinls +/usr/bin/xrdreplay +/usr/share/man/man1/xrdadler32.1 +/usr/share/man/man1/xrdcopy.1 +/usr/share/man/man1/xrdcp.1 +/usr/share/man/man1/xrdfs.1 +/usr/share/man/man1/xrdgsiproxy.1 +/usr/share/man/man1/xrdgsitest.1 +/usr/share/man/man1/xrdmapc.1 diff --git a/packaging/debian/xrootd-clients.install b/debian/xrootd-clients.install similarity index 100% rename from packaging/debian/xrootd-clients.install rename to debian/xrootd-clients.install diff --git a/packaging/debian/xrootd-devel.install b/debian/xrootd-devel.install similarity index 100% rename from packaging/debian/xrootd-devel.install rename to debian/xrootd-devel.install diff --git a/debian/xrootd-doc.install b/debian/xrootd-doc.install new file mode 100644 index 00000000000..5c1b34c5aa5 --- /dev/null +++ b/debian/xrootd-doc.install @@ -0,0 +1,2 @@ +doxydoc/html /usr/share/doc/xrootd +bindings/python/docs/build/python /usr/share/doc/xrootd diff --git a/debian/xrootd-fuse.install b/debian/xrootd-fuse.install new file mode 100644 index 00000000000..639e78d6f5f --- /dev/null +++ b/debian/xrootd-fuse.install @@ -0,0 +1,2 @@ +/usr/bin/xrootdfs +/usr/share/man/man1/xrootdfs.1 diff --git a/packaging/debian/xrootd-libs.install b/debian/xrootd-libs.install similarity index 100% rename from packaging/debian/xrootd-libs.install rename to debian/xrootd-libs.install diff --git a/debian/xrootd-plugins.install b/debian/xrootd-plugins.install new file mode 100644 index 00000000000..6effd2f0662 --- /dev/null +++ b/debian/xrootd-plugins.install @@ -0,0 +1,12 @@ +/usr/lib/*/libXrdCksCalczcrc32-5.so +/usr/lib/*/libXrdCryptossl-5.so +/usr/lib/*/libXrdSec-5.so +/usr/lib/*/libXrdSecProt-5.so +/usr/lib/*/libXrdSecgsi-5.so +/usr/lib/*/libXrdSecgsiAUTHZVO-5.so +/usr/lib/*/libXrdSecgsiGMAPDN-5.so +/usr/lib/*/libXrdSeckrb5-5.so +/usr/lib/*/libXrdSecpwd-5.so +/usr/lib/*/libXrdSecsss-5.so +/usr/lib/*/libXrdSecunix-5.so +/usr/lib/*/libXrdSecztn-5.so diff --git a/debian/xrootd-plugins.lintian-overrides b/debian/xrootd-plugins.lintian-overrides new file mode 100644 index 00000000000..a18591c49dc --- /dev/null +++ b/debian/xrootd-plugins.lintian-overrides @@ -0,0 +1,5 @@ +# These are plugins - no soname on purpose +sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] + +# False positive +library-not-linked-against-libc [usr/lib/*/libXrdCksCalczcrc32-5.so] diff --git a/packaging/debian/xrootd-private-devel.install b/debian/xrootd-private-devel.install similarity index 100% rename from packaging/debian/xrootd-private-devel.install rename to debian/xrootd-private-devel.install diff --git a/debian/xrootd-scitokens-plugins.docs b/debian/xrootd-scitokens-plugins.docs new file mode 100644 index 00000000000..ccd73502fdd --- /dev/null +++ b/debian/xrootd-scitokens-plugins.docs @@ -0,0 +1 @@ +src/XrdSciTokens/README.md diff --git a/debian/xrootd-scitokens-plugins.install b/debian/xrootd-scitokens-plugins.install new file mode 100644 index 00000000000..f285abbad81 --- /dev/null +++ b/debian/xrootd-scitokens-plugins.install @@ -0,0 +1 @@ +/usr/lib/*/libXrdAccSciTokens-5.so diff --git a/debian/xrootd-scitokens-plugins.lintian-overrides b/debian/xrootd-scitokens-plugins.lintian-overrides new file mode 100644 index 00000000000..754468af41c --- /dev/null +++ b/debian/xrootd-scitokens-plugins.lintian-overrides @@ -0,0 +1,2 @@ +# These are plugins - no soname on purpose +sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] diff --git a/packaging/debian/xrootd-server-devel.install b/debian/xrootd-server-devel.install similarity index 100% rename from packaging/debian/xrootd-server-devel.install rename to debian/xrootd-server-devel.install diff --git a/packaging/debian/xrootd-server-libs.install b/debian/xrootd-server-libs.install similarity index 100% rename from packaging/debian/xrootd-server-libs.install rename to debian/xrootd-server-libs.install diff --git a/debian/xrootd-server-plugins.install b/debian/xrootd-server-plugins.install new file mode 100644 index 00000000000..6e8777e4157 --- /dev/null +++ b/debian/xrootd-server-plugins.install @@ -0,0 +1,17 @@ +/usr/lib/*/libXrdBlacklistDecision-5.so +/usr/lib/*/libXrdBwm-5.so +/usr/lib/*/libXrdCmsRedirectLocal-5.so +/usr/lib/*/libXrdFileCache-5.so +/usr/lib/*/libXrdHttp-5.so +/usr/lib/*/libXrdHttpTPC-5.so +/usr/lib/*/libXrdMacaroons-5.so +/usr/lib/*/libXrdN2No2p-5.so +/usr/lib/*/libXrdOfsPrepGPI-5.so +/usr/lib/*/libXrdOssCsi-5.so +/usr/lib/*/libXrdOssSIgpfsT-5.so +/usr/lib/*/libXrdPfc-5.so +/usr/lib/*/libXrdPss-5.so +/usr/lib/*/libXrdSsi-5.so +/usr/lib/*/libXrdSsiLog-5.so +/usr/lib/*/libXrdThrottle-5.so +/usr/lib/*/libXrdXrootd-5.so diff --git a/debian/xrootd-server-plugins.lintian-overrides b/debian/xrootd-server-plugins.lintian-overrides new file mode 100644 index 00000000000..754468af41c --- /dev/null +++ b/debian/xrootd-server-plugins.lintian-overrides @@ -0,0 +1,2 @@ +# These are plugins - no soname on purpose +sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] diff --git a/debian/xrootd-server.install b/debian/xrootd-server.install new file mode 100644 index 00000000000..a75d81b9ec3 --- /dev/null +++ b/debian/xrootd-server.install @@ -0,0 +1,31 @@ +/usr/bin/cconfig +/usr/bin/cmsd +/usr/bin/frm_admin +/usr/bin/frm_purged +/usr/bin/frm_xfragent +/usr/bin/frm_xfrd +/usr/bin/mpxstats +/usr/bin/wait41 +/usr/bin/xrdacctest +/usr/bin/xrdpfc_print +/usr/bin/xrdpwdadmin +/usr/bin/xrdsssadmin +/usr/bin/xrootd +/usr/share/man/man8/cmsd.8 +/usr/share/man/man8/frm_admin.8 +/usr/share/man/man8/frm_purged.8 +/usr/share/man/man8/frm_xfragent.8 +/usr/share/man/man8/frm_xfrd.8 +/usr/share/man/man8/mpxstats.8 +/usr/share/man/man8/xrdpfc_print.8 +/usr/share/man/man8/xrdpwdadmin.8 +/usr/share/man/man8/xrdsssadmin.8 +/usr/share/man/man8/xrootd.8 +/usr/share/xrootd/utils +/lib/systemd/system/* +/usr/lib/tmpfiles.d/xrootd.conf +/etc/logrotate.d/xrootd +/etc/xrootd/config.d +/etc/xrootd/*.cfg +/var/log/xrootd +/var/spool/xrootd diff --git a/debian/xrootd-server.postinst b/debian/xrootd-server.postinst new file mode 100644 index 00000000000..e08626eb245 --- /dev/null +++ b/debian/xrootd-server.postinst @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e + +if test "$1" = "configure" -o "$1" = "reconfigure" ; then + + getent group xrootd > /dev/null || \ + addgroup --quiet --system xrootd + + getent passwd xrootd > /dev/null || \ + adduser --quiet --system --home /var/spool/xrootd --shell /bin/false \ + --ingroup xrootd --disabled-password --disabled-login \ + --gecos "XRootD runtime user" xrootd + + chown xrootd:xrootd /etc/xrootd/*.cfg + chown xrootd:xrootd /var/log/xrootd + chown xrootd:xrootd /var/spool/xrootd + +fi + +#DEBHELPER# diff --git a/debian/xrootd-voms-plugins.install b/debian/xrootd-voms-plugins.install new file mode 100644 index 00000000000..331ebced9ad --- /dev/null +++ b/debian/xrootd-voms-plugins.install @@ -0,0 +1,5 @@ +/usr/lib/*/libXrdVoms-5.so +/usr/lib/*/libXrdHttpVOMS-5.so +/usr/lib/*/libXrdSecgsiVOMS-5.so +/usr/share/man/man1/libXrdVoms.1 +/usr/share/man/man1/libXrdSecgsiVOMS.1 diff --git a/debian/xrootd-voms-plugins.lintian-overrides b/debian/xrootd-voms-plugins.lintian-overrides new file mode 100644 index 00000000000..754468af41c --- /dev/null +++ b/debian/xrootd-voms-plugins.lintian-overrides @@ -0,0 +1,2 @@ +# These are plugins - no soname on purpose +sharedobject-in-library-directory-missing-soname [usr/lib/*/libXrd*-5.so] diff --git a/docker/.dockerignore b/docker/.dockerignore new file mode 100644 index 00000000000..8fce603003c --- /dev/null +++ b/docker/.dockerignore @@ -0,0 +1 @@ +data/ diff --git a/docker/build/Dockerfile.alma8 b/docker/build/Dockerfile.alma8 new file mode 100644 index 00000000000..06aad45a888 --- /dev/null +++ b/docker/build/Dockerfile.alma8 @@ -0,0 +1,32 @@ +FROM almalinux:8 + +# Install tools necessary for RPM development +RUN dnf install -y dnf-plugins-core epel-release rpmdevtools sudo \ + && dnf config-manager --set-enabled powertools + +# Create xrootd user to avoid running tests as root +RUN groupadd xrootd && useradd -g xrootd -m xrootd + +USER xrootd +WORKDIR /home/xrootd + +# Create directory tree for building RPMs +RUN rpmdev-setuptree + +# XRootD source tarball must be created in the +# current directory in order to build this image +COPY xrootd.tar.gz rpmbuild/SOURCES + +# Extract spec file to build RPMs +RUN tar xzf rpmbuild/SOURCES/xrootd.tar.gz --strip-components=1 xrootd/xrootd.spec + +USER root + +# Install build dependencies with dnf +RUN dnf builddep -y xrootd.spec + +# Build RPMS for XRootD +RUN sudo -u xrootd rpmbuild -bb --with git xrootd.spec + +# Install RPMS +RUN yum install -y rpmbuild/RPMS/*/*.rpm diff --git a/docker/build/Dockerfile.alma9 b/docker/build/Dockerfile.alma9 new file mode 100644 index 00000000000..d44a781858c --- /dev/null +++ b/docker/build/Dockerfile.alma9 @@ -0,0 +1,32 @@ +FROM almalinux:9 + +# Install tools necessary for RPM development +RUN dnf install -y dnf-plugins-core epel-release rpmdevtools sudo \ + && dnf config-manager --set-enabled crb + +# Create xrootd user to avoid running tests as root +RUN groupadd xrootd && useradd -g xrootd -m xrootd + +USER xrootd +WORKDIR /home/xrootd + +# Create directory tree for building RPMs +RUN rpmdev-setuptree + +# XRootD source tarball must be created in the +# current directory in order to build this image +COPY xrootd.tar.gz rpmbuild/SOURCES + +# Extract spec file to build RPMs +RUN tar xzf rpmbuild/SOURCES/xrootd.tar.gz --strip-components=1 xrootd/xrootd.spec + +USER root + +# Install build dependencies with dnf +RUN dnf builddep -y xrootd.spec + +# Build RPMS for XRootD +RUN sudo -u xrootd rpmbuild -bb --with git xrootd.spec + +# Install RPMS +RUN yum install -y rpmbuild/RPMS/*/*.rpm diff --git a/docker/build/Dockerfile.centos7 b/docker/build/Dockerfile.centos7 new file mode 100644 index 00000000000..984fad32478 --- /dev/null +++ b/docker/build/Dockerfile.centos7 @@ -0,0 +1,32 @@ +FROM centos:7 + +# Install tools necessary for RPM development +RUN yum install -y centos-release-scl epel-release git \ + && yum install -y epel-rpm-macros rpmdevtools sudo yum-utils + +# Create xrootd user to avoid running tests as root +RUN groupadd xrootd && useradd -g xrootd -m xrootd + +USER xrootd +WORKDIR /home/xrootd + +# Create directory tree for building RPMs +RUN rpmdev-setuptree + +# XRootD source tarball must be created in the +# current directory in order to build this image +COPY xrootd.tar.gz rpmbuild/SOURCES + +# Extract spec file to build RPMs +RUN tar xzf rpmbuild/SOURCES/xrootd.tar.gz --strip-components=1 xrootd/xrootd.spec + +USER root + +# Install build dependencies with yum +RUN yum-builddep -y xrootd.spec + +# Build RPMS for XRootD +RUN sudo -u xrootd rpmbuild -bb --with git xrootd.spec + +# Install RPMS +RUN yum install -y rpmbuild/RPMS/*/*.rpm diff --git a/docker/build/Dockerfile.debian b/docker/build/Dockerfile.debian new file mode 100644 index 00000000000..b375f81c593 --- /dev/null +++ b/docker/build/Dockerfile.debian @@ -0,0 +1,31 @@ +ARG version=12 +FROM debian:$version + +RUN apt update +RUN apt install -y build-essential devscripts equivs sudo + +# Create xrootd user to avoid running tests as root +RUN groupadd xrootd && useradd -g xrootd -m xrootd + +USER xrootd +WORKDIR /home/xrootd + +# XRootD source tarball must be created in the +# current directory in order to build this image +COPY xrootd.tar.gz . + +# Extract tarball +RUN tar xzf xrootd.tar.gz + +USER root +WORKDIR /home/xrootd/xrootd + +# Install build dependencies with dnf +RUN echo yes | mk-build-deps --install --remove debian/control + +# Build DEB packages for XRootD +RUN export VERSION=$(sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./' < VERSION) \ + && sudo -u xrootd dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' \ + && sudo -u xrootd debuild --no-tgz-check --no-sign -b + +RUN apt install -y ../*.d*eb diff --git a/docker/build/Dockerfile.fedora b/docker/build/Dockerfile.fedora new file mode 100644 index 00000000000..6c11a2b91c7 --- /dev/null +++ b/docker/build/Dockerfile.fedora @@ -0,0 +1,32 @@ +ARG version=rawhide +FROM fedora:$version + +# Install tools necessary for RPM development +RUN dnf install -y dnf-plugins-core rpmdevtools sudo + +# Create xrootd user to avoid running tests as root +RUN groupadd xrootd && useradd -g xrootd -m xrootd + +USER xrootd +WORKDIR /home/xrootd + +# Create directory tree for building RPMs +RUN rpmdev-setuptree + +# XRootD source tarball must be created in the +# current directory in order to build this image +COPY xrootd.tar.gz rpmbuild/SOURCES + +# Extract spec file to build RPMs +RUN tar xzf rpmbuild/SOURCES/xrootd.tar.gz --strip-components=1 xrootd/xrootd.spec + +USER root + +# Install build dependencies with dnf +RUN dnf builddep -y xrootd.spec + +# Build RPMS for XRootD +RUN sudo -u xrootd rpmbuild -bb --with git xrootd.spec + +# Install RPMS +RUN yum install -y rpmbuild/RPMS/*/*.rpm diff --git a/docker/build/Dockerfile.ubuntu b/docker/build/Dockerfile.ubuntu new file mode 100644 index 00000000000..612a748c76a --- /dev/null +++ b/docker/build/Dockerfile.ubuntu @@ -0,0 +1,33 @@ +ARG version=22.04 +FROM ubuntu:$version + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update +RUN apt install -y build-essential devscripts equivs sudo + +# Create xrootd user to avoid running tests as root +RUN groupadd xrootd && useradd -g xrootd -m xrootd + +USER xrootd +WORKDIR /home/xrootd + +# XRootD source tarball must be created in the +# current directory in order to build this image +COPY xrootd.tar.gz . + +# Extract tarball +RUN tar xzf xrootd.tar.gz + +USER root +WORKDIR /home/xrootd/xrootd + +# Install build dependencies with dnf +RUN echo yes | mk-build-deps --install --remove debian/control + +# Build DEB packages for XRootD +RUN export VERSION=$(sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./' < VERSION) \ + && sudo -u xrootd dch --create --package xrootd -v ${VERSION} -M 'XRootD automated build.' \ + && sudo -u xrootd debuild --no-tgz-check --no-sign -b + +RUN apt install -y ../*.d*eb diff --git a/docker/builds/DockerfileCentos7 b/docker/builds/DockerfileCentos7 deleted file mode 100644 index af530bd9bde..00000000000 --- a/docker/builds/DockerfileCentos7 +++ /dev/null @@ -1,14 +0,0 @@ -FROM centos:7 -MAINTAINER Michal Simon , CERN, 2015 - -USER root -RUN yum install -y epel-release -RUN yum install -y gcc-c++ rpm-build which git python-srpm-macros centos-release-scl vim -RUN git clone https://github.com/xrootd/xrootd -WORKDIR /xrootd/packaging -RUN ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" -RUN yum-builddep -y *.src.rpm -RUN rm -f *src.rpm -WORKDIR /xrootd/build -ENTRYPOINT scl enable devtoolset-7 bash - diff --git a/docker/builds/DockerfileCentos8 b/docker/builds/DockerfileCentos8 deleted file mode 100644 index c5b03534987..00000000000 --- a/docker/builds/DockerfileCentos8 +++ /dev/null @@ -1,16 +0,0 @@ -FROM centos:8 -MAINTAINER Michal Simon , CERN, 2015 - -USER root -RUN dnf install -y epel-release -RUN dnf install -y gcc-c++ rpm-build which git python-srpm-macros vim dnf-plugins-core -RUN dnf config-manager --set-enabled powertools -RUN git clone https://github.com/xrootd/xrootd -WORKDIR /xrootd/packaging -RUN ./makesrpm.sh --define "_with_python3 1" --define "_with_tests 1" --define "_with_xrdclhttp 1" --define "_with_scitokens 1" --define "_with_isal 1" -RUN dnf builddep -y *.src.rpm -RUN dnf -y update libarchive -RUN rm -f *src.rpm -WORKDIR /xrootd/build -ENTRYPOINT bash - diff --git a/docker/builds/centos7_buildenv.sh b/docker/builds/centos7_buildenv.sh deleted file mode 100755 index 79d5705c667..00000000000 --- a/docker/builds/centos7_buildenv.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker build -t xrootd/centos7/build - < DockerfileCentos7 -docker run -it --rm xrootd/centos7/build diff --git a/docker/builds/centos8_buildenv.sh b/docker/builds/centos8_buildenv.sh deleted file mode 100755 index 28dc84d8370..00000000000 --- a/docker/builds/centos8_buildenv.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker build -t xrootd/centos8/build - < DockerfileCentos8 -docker run -it --rm xrootd/centos8/build diff --git a/docker/xrd-docker b/docker/xrd-docker new file mode 100755 index 00000000000..4ec7367318f --- /dev/null +++ b/docker/xrd-docker @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +trap 'exit 1' TERM KILL INT QUIT ABRT + +: ${DOCKER:=$(command -v docker || command -v podman)} + +build() { + OS=${1:-centos7} + [[ -f xrootd.tar.gz ]] || package + [[ -f build/Dockerfile.${OS} ]] || die "unknwon OS: $OS" + ${DOCKER} build -f build/Dockerfile.${OS} -t xrootd -t xrootd:${OS} . +} + + +buildx() { + OS=${1:-fedora} + ARCH=${2:-amd64} + ARCH=${ARCH/linux\/} + [[ -f xrootd.tar.gz ]] || package + [[ -f build/Dockerfile.${OS} ]] || die "unknwon OS: $OS" + ${DOCKER} buildx build --platform linux/${ARCH} -f build/Dockerfile.${OS} -t xrootd:${OS}-${ARCH/\/} . +} + +qemu() { + ${DOCKER} run --rm --privileged multiarch/qemu-user-static --reset -p yes +} + +clean() { + rm -f xrootd.tar.gz +} + +die() { + echo "$(basename $BASH_ARGV0): error: $@" + exit 1 +} + +package() { + REPODIR=$(git rev-parse --show-toplevel) + VERSION=$(git describe ${1:-HEAD}) + echo Creating tarball for XRootD ${VERSION} + pushd ${REPODIR} >/dev/null + git archive --prefix=xrootd/ -o xrootd.tar.gz ${1:-HEAD} + popd >/dev/null + mv ${REPODIR}/xrootd.tar.gz . +} + +usage() { + echo $(basename $BASH_ARGV0) [COMMAND] [ARGS] + echo + echo COMMANDS: + echo + echo " clean -- remove tarball created by package command" + echo " package [VERSION] -- create xrootd.tar.gz tarball (VERSION=HEAD by default)" + echo " build [OS] -- build docker image based on OS: centos7 (default), alma8, alma9" + echo " buildx [OS] [ARCH] -- cross-build docker image based on OS/ARCH pair. Supported architectures" + echo " are amd64, aarch64, ppc64le, s390x (big-endian). Default OS is fedora." + echo " You can see supported platforms with docker buildx inspect --bootstrap." + echo " qemu -- setup QEMU to be able to run cross-builds with buildx command." + echo + echo " Note: The test suite runs automatically during the container builds" + echo +} + +[[ $# == 0 ]] && usage && exit 0 + +CMD=$1 +shift +[[ $(type -t ${CMD}) == "function" ]] || die "unknown command: ${CMD}" +cd $(dirname "${BASH_SOURCE[0]}") || die "cannot change directory" +$CMD $@ diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt new file mode 100644 index 00000000000..d0ce545a9b8 --- /dev/null +++ b/docs/CMakeLists.txt @@ -0,0 +1,3 @@ + +add_subdirectory(man) + diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 00000000000..4f3b311a485 --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,235 @@ +## XRootD Development Model and How to Contribute + +### Versioning + +XRootD software releases are organized into major, minor and patch versions, +with the intent that installing minor version releases with the same major +version do not take long to perform, cause significant downtime, or break +dependent software. New major versions can be more disruptive, and may +substantially change or remove software components. Releases are assigned +version numbers, such as "5.6.0", where: + +* The first number designates a major version. Major versions may introduce + binary incompatibility with previous major versions and may require code + dependent on libraries in the new major version to be recompiled. Generally, + such requirements are limited to code that enhances XRootD functionality (e.g. + plug-ins). User application code that only uses public APIs should continue + to work unchanged. Consequently, major versions are infrequent and are + introduced approximately every 5 years. + +* The second number increments within the major version and designates a minor + version. Minor versions introduce new features within a major version. They + are binary compatible with all versions within the major version and occur as + often as needed to address community needs. On average, there are a few minor + versions per year. + +* The last digit increments whenever changes are applied to a minor version to + fix problems. These occur at random frequency, as often as necessary to fix + problems. Since patch versions represent the minimum change necessary to fix + a problem they provide the forward path for problem resolution. + +* When the first number increments, the second and third numbers are reset to + zero and when the second number increments the third number is reset to zero. + +* A fourth number may be added by EOS as indication that the version of XRootD + used by EOS has been patched after the official release. Patches introduced + in an intermediate release for EOS will be likely included into the following + patch release, unless it is a temporary fix affecting only EOS. + +#### Library versions + +When a library evolves compatibly: existing interfaces are preserved, but new +ones are added the library’s minor version number must be incremented. Since +nothing has been done that would break applications constructed earlier, it is +OK for older applications to be linked with the newer library at run-time. + +If the interfaces in a library shared object change incompatibly, then the major +revision number associated with the library must be incremented. Doing so will +cause run-time linking errors for the applictions constructed with the older +versions of the library and thus will prevent them from running, as opposed to +crashing in an uncontrollable way. + +More information on library versioning is available +[here](https://www.usenix.org/legacy/publications/library/proceedings/als00/2000papers/papers/full_papers/browndavid/browndavid_html/) +and +[here](https://www.akkadia.org/drepper/dsohowto.pdf). + +The project policy is that a change to public interfaces (as defined in the +installed headers) requires a major release - bumping the major version number. + +### Releases and Release Procedure + +Feature releases with current developments will normally be built a few times +per year. Each `master` release is preceded by one or more release candidates +that get tested for bugs and deployment issues in a possibly wide range of +environments. When the release candidates are deemed sufficiently stable, then +the final release is built. + +In addition to the `master` or "feature" releases, "bug fix" releases may be built +whenever needed. These are for bug fixes only, so they normally should not need +release candidates (due to the reduced need for additional testing). + +RPM packages are built for each release, including release candidates. All the +packages are pushed to the testing yum repository. Additionally, all the bug fix +releases and all the final `master` releases are pushed to the stable repository. +See the [download](https://xrootd.slac.stanford.edu/dload.html) page for details. + +### Stable and Develoment Branches + +Beginning with XRootD 5.6.0, the development model is based on two long-term +branches: `master`, and `devel`. + +The `master` branch is the stable branch. It contains released versions of +XRootD and may also contain unreleased bug fixes which do not require a new +minor release. Each patch release for a given major+minor series is created from +the `master` branch by adding any required bug fixes from the `devel` branch to +the `master` branch and tagging a new release, such that all XRootD releases may +be found linearly in git history. + +The `devel` branch is the development branch where all new features and other +developments happen. Each new feature release is created by rebasing, then +(perharps partially) merging the `devel` branch into the `master` branch, then +tagging the relase on `master`. The `devel` branch will be kept current with the +`master` branch by rebasing it after each patch release, to ensure that all bug +fixes are always included in both `master` and `devel`. + +### Guidelines for Contributors + +This section provides guidelines for people who want to contribute +code to the XRootD project. It is adapted from git's own guidelines +for contributors, which can be found in their repository on GitHub at +https://github.com/git/git/blob/master/Documentation/SubmittingPatches. + +#### Deciding what to base your work on + +In general, always base your work on the oldest branch that your +change is relevant to. + +* A bug fix should be based on the latest release tag in general. If + the bug is not present there, then base it on `master`. Otherwise, + if it is only present on `devel`, or a feature branch, then base it + on the tip of `devel` or the relevant feature branch. + +* A new feature should be based on `devel` in general. If the new + feature depends on topics which are not yet merged, fork a branch + from the tip of `devel`, merge these topics to the branch, and work + on that branch. You can get an idea of how the branches relate to + each other with `git log --first-parent master..` or with + `git log --all --decorate --graph --oneline`. + +* Corrections and enhancements to a topic not yet merged into `devel` + should be based on the tip of that topic. Before merging, we recommend + cleaning up the history by squashing commits that are fixups for + earlier commits in the same branch rather than committing a bad change + and the fix for it in separate commits. This is important to preserve + the ability to use git bisect to find which commit introduced a bug. + +#### Make separate commits for logically separate changes + +In your commits, you should give an explanation for the change(s) that +is detailed enough so that a code reviewer can judge if it is a good +thing to do or not without reading the actual patch text to determine +how well the code actually does it. + +If your description is too long, that's probably a sign that the commit +should be split into finer grained pieces. That being said, patches which +plainly describe the things that help reviewers checking the patch and +future maintainers understand the code are very welcome. + +If you are fixing a bug, it would be immensely useful to include a test +demonstrating the problem being fixed, so that not only the problem is +avoided in the future, but also reviewers can more easily verify that the +proposed fix works as expected. Similarly, new features which come with +accompanying tests are much more likely to be reviewed and merged in a +timely fashion. + +When developing XRootD on your own fork, please make sure that the +existing test suite is not broken by any of your changes by pushing to +a branch in your own fork and checking the result of the GitHub Actions +runs. + +#### Describe your changes well + +The log message that explains your changes is just as important as the +changes themselves. The commit messages are the base for creating the +release notes for each release. Hence, each commit message should clearly +state whether it is a bug fix or a new feature whenever that is not +immediately obvious from the nature of the change itself. Moreover, it is +very important to explain not only _what_ your code does, but also _why_ +it does it. + +The first line of the commit message should be a short description of up +to about 50 characters (soft limit, hard limit at 80 characters), and +should skip the full stop. It is encouraged, although not necessary, to +include a tag which identifies the general area of code being modified, +for example "[Server]", "[XrdHttp]", etc. If in doubt, please check the +git log for similar files to see what are the current conventions. + +After the title sentence, you should include a blank line and then the +body of the commit message, which should be a meaningful description that + +* explains the problem the change tries to solve, i.e. what is wrong + with the current code without the change. + +* justifies the way the change solves the problem, i.e. why the + result with the change is better. + +* alternate solutions considered but discarded, if any. + +You should use the imperative to describe your changes, for example: +``` +Change default value of foo to 1024 +``` +instead of +``` +This commit changes the default value of foo to 1024 +``` +or +``` +Changed default default value of foo to 1024 +``` + +Examples of good commit messages: + +``` +Author: Andrew Hanushevsky +Date: Thu Jun 8 18:06:01 2023 -0700 + + [Server] Allow generic prepare plug-in to handle large responses, fixes #2023 +``` + +``` +Author: Brian Bockelman +Date: Sat Feb 18 13:15:49 2023 -0600 + + Map authentication failure to HTTP 401 + + The authentication failure error message was previously mapped to + HTTP 500 (internal server error). 401 Unauthorized (despite its name) + is what HTTP servers typically utilize for authentication problems. +``` + +#### References to other commits, issues, pull requests, etc + +Sometimes, it may be useful to refer to the pull request on GitHub, an +open issue which a commit fixes/closes, or simply an older commit which +may have introduced a regression fixed by the current change. When referring +to older commits, try to use the same format as produced by +``` +git show -s --pretty=reference +``` + +For issues, add a "Closes: #nnnn" or "Fixes: #nnnn" tag to the body of the +commit message (or, even better, to the pull request description). When +linking a change to a specific issue or pull request, please verify in the +GitHub website that the association actually worked. Depending on how you +phrase your message, this may not happen automatically. In that case, it is +also possible to use the "Development" side panel on the right to manually +create the connection between pull requests and issues. If you intend to +have your changes be part of a particular release which is not the next +release being planned, you may also mark your pull request for inclusion +in the desired release by using the "Milestone" side panel on the right. +This can be used as an alternative method of marking a change as "bug fix" +or "feature", depending on if it will only be included in the next patch +release or feature release. Any changes which require a major release must +be marked with the appropriate milestone. diff --git a/docs/INSTALL.md b/docs/INSTALL.md new file mode 100644 index 00000000000..b594ebcee2c --- /dev/null +++ b/docs/INSTALL.md @@ -0,0 +1,198 @@ +## Building and Installing XRootD from Source Code + +### XRootD Required and Optional Build Dependencies + +The required build-time dependencies are: bash, cmake, make, a C++ compiler with +support for C++14, kernel headers from the OS, and development packages for +libuuid, OpenSSL, and zlib. The optional features are shown in the table below +together with the dependencies required to enable them. + +| Feature | Dependencies | +|:----------------------|:-----------------------------------------| +| FUSE support | fuse-devel | +| HTTP support (client) | davix-devel | +| HTTP support (server) | libcurl-devel | +| Erasure coding | isa-l / autoconf automake libtool yasm | +| Kerberos5 | krb5-devel | +| Macaroons | json-c-devel libmacaroons-devel | +| Python bindings | python3-devel python3-setuptools | +| readline | readline-devel | +| SciTokens | scitokens-cpp-devel | +| systemd support | systemd-devel | +| VOMS | voms-devel | +| Testing | cppunit-devel gtest-devel | + +Other optional dependencies: tinyxml-devel, libxml2-devel. These have bundled +copies which are used if not found in the system. In the following sections, we +show how to install all available dependencies in most of the supported operating +systems. + +#### RPM-based distributions: RedHat, Fedora, CentOS, Alma, Rocky + +On RedHat Enterprise Linux and derivatives, all dependencies are available, +except for Intel's [isa-l](https://github.com/intel/isa-l) library. You may +build and install isa-l from source, but alternatively, you can simple install +isa-l dependencies (i.e. `autoconf`, `automake`, `libtool`, and `yasm`) and let +the bundled isa-l be built along XRootD. On Fedora, it's not necessary to +install the `epel-release` package, but on most others it is required, as some +dependencies are only available in [EPEL](https://docs.fedoraproject.org/en-US/epel/). +It may also be necessary to enable other repositories manually if they are not +already enabled by default, like the `PowerTools`, `crb`, or `scl` repositories. +On CentOS 7, this can be done by installing `centos-release-scl` in addition to +`epel-release`. The command to install all dependencies is shown below. You may, +however, need to replace `dnf` with `yum` or prepend `sudo` to this command, +depending on the distribution: + +```sh +dnf install \ + cmake \ + cppunit-devel \ + curl-devel \ + davix-devel \ + diffutils \ + file \ + fuse-devel \ + gcc-c++ \ + git \ + gtest-devel \ + json-c-devel \ + krb5-devel \ + libmacaroons-devel \ + libtool \ + libuuid-devel \ + libxml2-devel \ + make \ + openssl-devel \ + python3-devel \ + python3-setuptools \ + readline-devel \ + scitokens-cpp-devel \ + systemd-devel \ + tinyxml-devel \ + voms-devel \ + yasm \ + zlib-devel +``` + +On CentOS 7, the default compiler is too old, so `devtoolset-11` should be used +instead. It can be enabled afterwards with `source /opt/rh/devtoolset-11/enable`. +Moreover, `cmake` installs CMake 2.x on CentOS 7, so `cmake3` needs to be installed +instead and `cmake3`/`ctest3` used in the command line. + +#### DEB-based distrubutions: Debian 11, Ubuntu 22.04 + +On Debian 11 and Ubuntu 22.04, all necessary dependencies are available in the +system. In earlier versions, some of XRootD's optional features may have to be +disabled if their dependencies are not available to be installed via apt. + +```sh +apt install \ + cmake \ + davix-dev \ + g++ \ + libcppunit-dev \ + libcurl4-openssl-dev \ + libfuse-dev \ + libgtest-dev \ + libisal-dev \ + libjson-c-dev \ + libkrb5-dev \ + libmacaroons-dev \ + libreadline-dev \ + libscitokens-dev \ + libssl-dev \ + libsystemd-dev \ + libtinyxml-dev \ + libxml2-dev \ + make \ + pkg-config \ + python3-dev \ + python3-setuptools \ + uuid-dev \ + voms-dev \ + zlib1g-dev +``` + +### Homebrew on macOS + +On macOS, XRootD is available to install via Homebrew. We recommend using it to +install dependencies as well when building XRootD from source: + +```sh +brew install \ + cmake \ + cppunit \ + curl \ + davix \ + gcc \ + googletest \ + isa-l \ + krb5 \ + libxml2 \ + libxcrypt \ + make \ + openssl@1.1 \ + pkg-config \ + python@3.11 \ + readline \ + zlib \ +``` + +Homebrew is also available on Linux, where `utils-linux` is required as +an extra dependency since uuid symbols are not provided by the kernel like +on macOS. On Linux, `libfuse@2` may be installed to enable FUSE support. + +## Building from Source Code with CMake + +XRootD uses [CMake](https://cmake.org) as its build generator. CMake +is used during configuration to generate the actual build system that +is used to build the project with a build tool like `make` or `ninja`. +If you are new to CMake, we recommend reading the official +[tutorial](https://cmake.org/cmake/help/latest/guide/tutorial/index.html) +which provides a step-by-step guide on how to get started with CMake. +For the anxious user, assuming the repository is cloned into the `xrootd` +directory under the current working directory, the basic workflow is + +```sh +cmake -S xrootd -B xrootd_build +cmake --build xrootd_build --parallel +cmake --install xrootd_build +``` + +If you'd like to install somewhere other than the default of `/usr/local`, +then you need to pass the option `-DCMAKE_INSTALL_PREFIX=` to +the first command, with the location where you'd like to install as argument. + +The table below documents the main build options that can be used to customize +the build: + +| CMake Option | Default | Description +|:-------------------|:--------|:-------------------------------------------------------------- +| `ENABLE_FUSE` | TRUE | Enable FUSE filesystem driver +| `ENABLE_HTTP` | TRUE | Enable HTTP component (XrdHttp) +| `ENABLE_KRB5` | TRUE | Enable Kerberos 5 authentication +| `ENABLE_MACAROONS` | TRUE | Enable Macaroons plugin (server only) +| `ENABLE_PYTHON` | TRUE | Enable building of the Python bindings +| `ENABLE_READLINE` | TRUE | Enable readline support in the commandline utilities +| `ENABLE_SCITOKENS` | TRUE | Enable SciTokens plugin (server only) +| `ENABLE_VOMS` | TRUE | Enable VOMS plug-in +| `ENABLE_XRDCLHTTP` | TRUE | Enable xrdcl-http plugin +| `ENABLE_XRDCL` | TRUE | Enable XRootD client +| `ENABLE_XRDEC` | FALSE | Enable support for erasure coding +| `ENABLE_ASAN` | FALSE | Build with adress sanitizer enabled +| `ENABLE_TSAN` | FALSE | Build with thread sanitizer enabled +| `ENABLE_TESTS` | FALSE | Enable unit tests +| `FORCE_ENABLED` | FALSE | Fail CMake configuration if enabled components cannot be built +| `XRDCL_LIB_ONLY` | FALSE | Build only the client libraries and necessary dependencies +| `XRDCL_ONLY` | FALSE | Build only the client and necessary dependencies +| `USE_SYSTEM_ISAL` | FALSE | Use isa-l library installed in the system + +### Running XRootD Tests + +After you've built the project, you should run the unit and integration tests +with CTest to ensure that they all pass. This can be done simply by running +`ctest` from the build directory. However, we also provide a CMake script to +automate more advanced testing, including enabling a coverage report, memory checking with +`valgrind`, and static analysis with `clang-tidy`. The script is named [test.cmake](../test.cmake) +and can be found in the top directory of the repository. Its usage is documented in the file +[TESTING.md](TESTING.md). diff --git a/docs/PreReleaseNotes.txt b/docs/PreReleaseNotes.txt deleted file mode 100644 index 33572e5648a..00000000000 --- a/docs/PreReleaseNotes.txt +++ /dev/null @@ -1,20 +0,0 @@ -====== -XRootD -====== - -Prerelease Notes -================ - -+ **New Features** - **[PSS]** Allow origin to be a directory of a locally mounted file system. - **Commit: 850a14f bb550ea - **[Server]** Add gsi option to display DN when it differs from entity name. - **Commit: 2630fe1 - **[Server]** Allow specfication of minimum and maximum creation mode. - **Commit: 8a6d7c0 - -+ **Major bug fixes** - -+ **Minor bug fixes** - -+ **Miscellaneous** diff --git a/docs/ReleaseNotes.txt b/docs/ReleaseNotes.txt index e393202c798..920407f317c 100644 --- a/docs/ReleaseNotes.txt +++ b/docs/ReleaseNotes.txt @@ -5,6 +5,287 @@ XRootD Release Notes ============= +------------- +Version 5.6.9 +------------- + ++ **Minor bug fixes** + **[Python]** Check list of files in prepare to ensure they are strings + **[Python]** Fix iteration over a file with Python3 + **[Python]** Use int for 'force' in File::Stat (#2208) + **[Utils]** Correct comparison that wrongly missed reaping certain directives + **[XrdCl]** Fix logic error when upgrading connections to TLS + **[XrdCl]** Stop Poller before TaskManager (fixes rare crashes at shutdown) + **[XrdHttpTPC]** Fix 500 server response code if X-Number-Of-Streams > 100 (issue #2186) + **[XrdSciTokens]** Add stat permissions to create, modify and write operations (issue #2185) + **[XrdSciTokens]** Allow creation of parent directories if necessary (#2184) + **[XrdSciTokens]** Fix bug when scope includes basepath or `/` (issue #2132) + ++ **Miscellaneous** + **[Tests]** Optimize cluster configuration to speedup tests + +------------- +Version 5.6.8 +------------- + ++ **Minor bug fixes** + **[RPM]** Create systemd tmpfiles at post-install step + **[XrdCl]** Only claim to be TLS capable if TLS initialization succeeds (issue #2020) + **[XrdCl]** Only consider an endpoint TLS-enabled if the connection is encrypted** + **[XrdCl]** Remove duplicates from URL list to avoid undefined behavior + **[XrdHttpTPC]** Fix infinite loop when scitags packet marking is enabled (issue #2192) + **[XrdPosix,XrdSecztn]** Fix build on FreeBSD (issue #2090) + **[XrdTls]** Fix automatic renewal of server certificate with OpenSSL>=1.1 (issue #1678) + ++ **Miscellaneous** + **[CMake]** Use CTest module in test.cmake and optionally submit to CDash + **[RPM]** Install the client as dependency of main RPM + **[Server]** Fix clang compile warnings + +------------- +Version 5.6.7 +------------- + ++ **Major bug fixes** + **[XrdCl]** Fix crash at teardown when using copies with multiple streams (issue #2164) + **[XrdSecsss]** Fix buffer overrun when serializing credentials (issue #2143) + ++ **Minor bug fixes** + **[XrdCl]** Fix TPC initialization to take into account control stream (issue #2164) + **[XrdPosix]** Fix ordering of debug levels in pss.setop DebugLevel (#2183) + **[XrdTpc]** Properly handle creation of packet marking handles when socket is not yet connected (#2179) + ++ **Miscellaneous** + **[XrdHeaders]** Install XrdSfsFAttr.hh as private header + +------------- +Version 5.6.6 +------------- + ++ **Major bug fixes** + **[XrdHttp]** Fix PostProcessHttpReq to take into account User-Agent setting (#2173) + ++ **Minor bug fixes** + **[Server]** Set TZ environment variable to avoid race conditions (issue #2107) + **[XrdCl]** Treat errOperationInterrupted as a recoverable error (issue #2169) + +------------- +Version 5.6.5 +------------- + ++ **Major bug fixes** + **[XrdTpc]** Fix potential segmentation fault when creating packet marking handle (issue #2161) + ++ **Minor bug fixes** + **[XrdSecgsi]** Fix compilation with GCC 14 (#2165) + **[XrdSys]** Include for BSD and GNU/Hurd (#2149) + ++ **Miscellaneous** + **[Server]** Align monitoring ID with HTTP (issue #2133) + **[XrdCrypto]** Skip check of our standard DH parameters (issue #2162) + **[XrdHttp]** Send User-Agent as part of monitoring info (#2154) + +------------- +Version 5.6.4 +------------- + ++ **Major bug fixes** + **[XrdHttp]** Fix segfault with macaroons (issue #2114) + **[XrdPss]** Fix segfault if pss.origin uses https protocol with no port (issue #2140) + ++ **Minor bug fixes** + **[CMake]** Fix include path in XRootDConfig.cmake (#2142) + **[Headers]** Fix header dependencies and missing includes/declarations (#2119) + **[Server]** Initialize pidFN to pidpath base directory if an error occurs + **[XrdCl]** Don't try to enable TCP_CORK in GNU/Hurd (#2115) + **[XrdCl]** Reapply fix for null-characters in error output (#2138, issue #1501) + **[XrdEc]** Fix alignment issues on SPARC (issue #2131) + **[XrdHttp,XrdNet]** Adapt Scitag min and max value to change in spec (#2139) + **[XrdPosix]** Fix compilation failure with latest MUSL libc + **[XrdSciTokens]** Initialize SecEntity.addrInfo to avoid SEGV (#2128) + **[XrdTls]** Switch from using a cert file to a cert chain file (issue #2126) + **[XrdZip]** Support big endian architectures in XrdZip (#2145) + ++ **Miscellaneous** + **[CMake]** Install CMake config file into lib/lib64 rather than share (#2116) + **[DEB/RPM]** Rewrite packaging for Debian and RHEL based distributions + **[Tests]** Convert tests to GoogleTest and run without containers (#2055, GSoC 2023) + **[Tests]** Other fixes and improvements to tests (#2115, #2129, #2130, #2137, #2141) + ++ **Known Issues** + **[XrdSciTokens]** In this release, as in previous ones, using scitokens with the ZTN + protocol may grant more access than should be allowed by the token scopes (issue #2121). + +------------- +Version 5.6.3 +------------- + ++ **Minor bug fixes** + **[CMake]** Export project version in CMake config (issue #2094) + **[CMake]** Find only XRootD matching XRootDConfig.cmake installation path + **[Python]** Do not use PEP517 by default, not supported on CentOS 7 + **[Server]** Call tzset() early to ensure thread-safety of localtime_r() and mktime() (issue #2107) + **[Server]** Correct maximum exp/act value in XrdNetPMark::getEA + **[Server]** Create environment file within adminpath (issue #2106) + **[Server]** Fix incorrect patch for authfile parsing (issue #2088) + **[Tests]** Skip server checksum query test on unsupported filesystems (issue #2096) + **[XrdCl]** Return an error if xrdfs rm fails to delete any file (issue #2097) + **[XrdCms]** Try to load blacklist even if some entries are invalid (issue #2092) + **[XrdEc]** Wait for pipeline including XrdCl::AppendFile() to finish (issue #2050) + **[XrdHttp]** Fix parsing of chunked PUT lengths (#2102, #2103) + ++ **Miscellaneous** + **[CMake]** Add extra debugging messages in XRootDConfig.cmake + **[CMake]** Handle components using more standard method + **[Misc]** Fix spelling errors reported by lintian (#2087) + **[Python]** Convert pyxrootd installation instructions to rst + **[Server]** Export ptr to full TLS context into the Xrd env + **[XrdCeph]** Align CMake requirement with main CMakeLists.txt + **[XrdHttp]** Implemented HTTP TPC Packet Marking (#2109) + **[XrdHttp]** Parse headers provided by the client in case-insensitive way when matching header2cgi keys (#2101) + **[XrdHttp]** Promote SciTag header if packet marking has been configured on the server (#2101) + **[XrdSciTokens]** Use configured CA path in SciTokens plugin if supported (#2095, #2112) + **[XrdTpc]** Differentiate error messages for push/pull TPC transfer modes (issue #2060) + +------------- +Version 5.6.2 +------------- + ++ **Major bug fixes** + **[XrdHttp]** Fix chunked PUT creating empty files (issue #2058) + ++ **Minor bug fixes** + **[CMake]** Update Findlibuuid.cmake to use correct include paths + **[Python]** Fix inclusion of markdown README file in documentation (#2057) + **[Server]** Align code with actual documentation for auth idspec (issue #2061) + **[XrdCl]** Fix flag check for append in XrdClZipArchive + **[XrdCl]** Fix promotion of root:// URLs to use TLS encryption (issue #2078) + **[XrdHttp]** Correct chunked response for GET with a byte range (issue #2076) + **[XrdHttp]** Refactor read issuing during GET and fix read vector too long (issue #1976) + **[XrdSciTokens]** Fix logic error in user mapping (issue #2056) + **[XrdSciTokens]** Update maximum header size and line length in INI files (issue #2074) + **[XrdSecgsi]** Fix crash of xrdgsitest when proxy is not already set + **[XrdSecztn]** Fix template for default ZTN token location (issue #2080) + **[XrdTls]** Change the thread-id returned to openssl 1.0 to improve performance (issue #2084) + **[XrdTls]** Insert CRLs containing critical extensions at the end of the bundle (issue #2065) + ++ **Miscellaneous** + **[CMake]** Always compile XrdOssCsi (compiled only with GCC before) + **[CMake]** Hide build output for isa-l to not confuse CTest + **[CMake]** Run tests in parallel and fail build when tests fail + **[Python]** Allow build customization via environment variable (issue #2062) + **[Python]** Check for Development.Module with CMake 3.18 and above + **[Server]** Add initialiser in one of the XrdScheduler constructors (#2081) + **[Server]** Default ffdest as per current pmark specification + **[Server]** Export readv comma separated limits via XRD_READV_LIMITS envar + **[Server]** Implement Linux epoll maxfd limit (#2063) + **[XrdClHttp] Add pgWrite support to the HTTP client plugin + **[XrdHttp]** Refactor request statemachine for HTTP GET requests (#2072) + **[XrdTls]** Refactor CASet and CRLSet to open the output file only once before the processing + +------------- +Version 5.6.1 +------------- + ++ **Minor bug fixes** + **[CMake]** Fix Findlibuuid.cmake to use kernel provided uuid on macOS + **[XrdCl]** Avoid race in postmaster QueryTransport + **[XrdCl]** Add missing argument in call to debug log message. + This fixes sporadic crashes seen in FTS when debug logging is enabled. + **[XrdCrypto]** Avoid race in GetCryptoFactory + ++ **Miscellaneous** + **[CMake]** Make sure Python is required in PyPI build + **[CMake]** Set RPATH that works for binaries and libraries on macOS + **[CMake,Python]** Handle RPATH for Python bindings on macOS + **[Python]** Use PEP517 by default when building Python bindings + +------------- +Version 5.6.0 +------------- + ++ **New Features** + **[CMake]** Modernization of build system, now requires CMake 3.16 + **[Client]** Add xrdfs cache subcommand to allow for cache evictions + **[Misc]** Add support for building with musl libc (issue #1645) + **[Python]** Modernization of build system, better support for creating binary wheels, + properly propagating CXXFLAGS (issues #1768, #1807, #1833, #1844, #2001, #2002) + **[Python]** Better handling of unicode strings in the API (issue #2011) + **[Server]** Add gsi option to display DN when it differs from entity name + **[Server]** Allow generic prepare plug-in to handle large responses (issue #2023) + **[Server]** Allow specfication of minimum and maximum creation mode (issue #649) + **[Server]** Make maxfd be configurable (default is 256k) (issue #2010) + **[Server]** Include token information in the monitoring stream (phase 1). + **[Xcache]** Implement a file evict function + **[Xcache,XrdCl]** Increase default number of parallel event loops to 10 (#2047) + **[XrdCl]** xrdcp: number of parallel copy jobs increased from 4 to 128 + **[XrdHttp]** Allow XRootD to return trailers indicating failure (#1912) + **[XrdHttp]** Denote Accept-Ranges in HEAD response (issue #1889) + **[XrdHttp]** Report cache object age for caching proxy mode (#1919) + **[XrdPss]** Allow origin to be a directory of a locally mounted file system + **[XrdSciTokens]** Implement ability to have the token username as a separate claim (#1978) + **[XrdSecgsi]** Use SHA-256 for signatures, and message digest algorithm (issues #1992, #2030) + **[XrdSecztn]** Allow option '-tokenlib none' to disable token validation (issue #1895) + **[XrdSecztn]** Allow to point to a token file using CGI '?xrd.ztn=tokenfile' (#1926) + ++ **Major bug fixes** + **[XrdHttp]** Fix SEGV in case request has object for opaque data but no content (#2007) + **[XrdSecgsi]** Fix memory leaks in GSI authentication (issue #2021) + ++ **Minor bug fixes** + **[Server]** Use correct value for testing vector size + **[XrdCl]** Fix off by one error in failure recovery check in parallel operation (issue #2040) + **[XrdCl]** Fix potential stream timeout when a new request is sent to an idle stream (issue #2042) + **[XrdCl]** Do not enforce TLS when --notlsok option is used in combination with root:// URL. + This allows falling back to e.g. Kerberos authentication on a server with ZTN plugin + enabled if the client has no certificates, hence not able to use TLS (issue #2020) + **[XrdEc]** Fix compilation issues and underlinking on macOS + **[XrdHttp]** Fix error returned when a client provides too many range requests (issue #2003) + **[XrdHttp]** Fix regression where performance markers were missing during an HTTP TPC transfer (#2017) + **[XrdHttp]** Return 404 instead of 500 error code on GET request on non-existent file (issue #2018) + **[XrdHttp]** Return 405 instead of 500 error code on deletion of non-empty directory (issue #1896) + **[XrdHttp]** Update HTTP header handling for chunked encoding and status trailer (#2009) + **[XrdTls]** Make sure TLS context is marked invalid if not properly constructed (issue #2020) + **[XrdTls]** Fix build failure with latest glibc (#2012) + ++ **Miscellaneous** + **[Apps]** Make xrdcrc32c consistent with xrdadler32 (issue #2045) + **[CMake]** Build option ENABLE_CRYPTO has been removed. OpenSSL is always required with XRootD 5 (issue #1827) + **[CMake]** New test.cmake script added to automate configure/build/test cycle + **[CMake]** Fix build with link-time optimizations on 32bit systems (issue #2032) + **[docs]** Update READMEs, contribution, installation, and testing documentation + **[Misc]** Fix warnings from Clang compiler (#1997) + **[Misc]** Add sandboxing settings to systemd service files (initially commented out) (issue #2033) + **[Server]** Also check for IPv6 ULA's to determine if an address is private + **[Tests]** New script xrd-docker added to automate running of dockerized tests (#1974) + **[XProtocol]** Add fallthrough statement for ENOTEMPTY errno code mapping + **[XRootD] ** Update code to no longer rely on using namespace std; (needed to support C++17) + **[XrdCeph]** Submodule merged back into main repository (#2008) + **[XrdCeph]** Minor build system updates and integration with main repository + **[XrdCrypto]** Switch to a fixed set of DH parameters compatible with older OpenSSL (issue #2014) + +------------- +Version 5.5.5 +------------- + ++ **Major bug fixes** + **[HTTP]** Initialize SecEntity.addrInfo to avoid SEGV (issue 1986) + **[Server]** Allow XrdXrootdFile::Serialize() to be used to wait more than once (issue 1995) + **[Server]** Correct file handle returned when reusing it from external table + **[XrdCl]** Fix client crash on early closed connection (issue 1934) + ++ **Minor bug fixes** + **[Server]** Use correct format to print size_t (issue 1989) + **[XrdApps]** Let XrdClProxyPlugin work with PgRead (issue 1993) + **[XrdCl]** Avoid possibility of Channel unregistering the wrong task (issue 1883) + **[XrdCl]** Fix ZipArchive issuing VectorWrite which is too large during CloseArchive (issue 2004) + **[XrdSys]** Avoid memory leak when overwriting the default message for EBADE + ++ **Miscellaneous** + **[CMake]** Adjust build rules to not depend on scitokenscpp to build libXrdSecztn + **[Server,XrdPosix,XrdSecgsi]** Fix compile warnings with Clang compiler + **[XrdPosix]** Fix build with Clang and _FORTIFY_SOURCE enabled (issue 1975) + ------------- Version 5.5.4 ------------- diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 00000000000..8bbdf98b08d --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,360 @@ +## Configuring and Running XRootD tests with CTest + +XRootD tests are divided into two main categories: unit and integration +tests that can be run directly with CTest, and containerized tests that +are required to be run from within a container built with docker or podman. +This document describes how to run the former, that is, the tests that are +run just with CTest. This document assumes you are already familiar with +how to build XRootD from source. If you need instructions on how to do that, +please see the [INSTALL.md](INSTALL.md) file. There you will also find a full +list of optional features and which dependencies are required to enable them. + +### Enabling tests during CMake configuration + +XRootD unit and integration tests are enabled via the CMake configuration +option `-DENABLE_TESTS=ON`. Unit and integration tests may depend on CppUnit +or GoogleTest (a migration from CppUnit to GoogleTest is ongoing). Therefore, +the development packages for CppUnit and GoogleTest (i.e. `cppunit-devel` and +`gtest-devel` on RPM-based distributions) are needed in order to enable all +available tests. Here we discuss how to use the [test.cmake](../test.cmake) +CTest script to run all steps to configure and build the project, then run all +tests using CTest. The script [test.cmake](../test.cmake) can be generically +called from the top directory of the repository as shown below + +```sh +xrootd $ ctest -V -S test.cmake +-- Using CMake cache file config.cmake +Run dashboard with model Experimental + Source directory: xrootd + Build directory: xrootd/build + Reading ctest configuration file: xrootd/CTestConfig.cmake + Site: example.cern.ch (Linux - x86_64) + Build name: Linux GCC 12.3.1 RelWithDebInfo + Use Experimental tag: 20230622-0712 + Updating the repository: xrootd + Use GIT repository type + Old revision of repository is: 6fce466a5f9b369f45ef2592c2ae246de1f13103 + New revision of repository is: 6fce466a5f9b369f45ef2592c2ae246de1f13103 + Gathering version information (one . per revision): + +Configure project + Each . represents 1024 bytes of output + ..... Size of output: 4K +Build project + Each symbol represents 1024 bytes of output. + '!' represents an error and '*' a warning. + .................................................. Size: 49K + .. Size of output: 52K + 0 Compiler errors + 0 Compiler warnings +Test project xrootd/build + Start 1: XrdCl::URLTest.LocalURLs + 1/23 Test #1: XrdCl::URLTest.LocalURLs ..................... Passed 0.01 sec + Start 2: XrdCl::URLTest.RemoteURLs + 2/23 Test #2: XrdCl::URLTest.RemoteURLs .................... Passed 0.12 sec + Start 3: XrdCl::URLTest.InvalidURLs + 3/23 Test #3: XrdCl::URLTest.InvalidURLs ................... Passed 0.01 sec + Start 4: XrdHttpTests.checksumHandlerTests + 4/23 Test #4: XrdHttpTests.checksumHandlerTests ............ Passed 0.01 sec + Start 5: XrdHttpTests.checksumHandlerSelectionTest + 5/23 Test #5: XrdHttpTests.checksumHandlerSelectionTest .... Passed 0.01 sec + Start 6: XrdCl::Poller + 6/23 Test #6: XrdCl::Poller ................................ Passed 5.01 sec + Start 7: XrdCl::Socket + 7/23 Test #7: XrdCl::Socket ................................ Passed 0.02 sec + Start 8: XrdCl::Utils + 8/23 Test #8: XrdCl::Utils ................................. Passed 8.01 sec + Start 9: XrdEc::AlignedWriteTest + 9/23 Test #9: XrdEc::AlignedWriteTest ...................... Passed 0.06 sec + Start 10: XrdEc::SmallWriteTest +10/23 Test #10: XrdEc::SmallWriteTest ........................ Passed 0.06 sec + Start 11: XrdEc::BigWriteTest +11/23 Test #11: XrdEc::BigWriteTest .......................... Passed 0.05 sec + Start 12: XrdEc::VectorReadTest +12/23 Test #12: XrdEc::VectorReadTest ........................ Passed 0.06 sec + Start 13: XrdEc::IllegalVectorReadTest +13/23 Test #13: XrdEc::IllegalVectorReadTest ................. Passed 0.06 sec + Start 14: XrdEc::AlignedWrite1MissingTest +14/23 Test #14: XrdEc::AlignedWrite1MissingTest .............. Passed 0.06 sec + Start 15: XrdEc::AlignedWrite2MissingTest +15/23 Test #15: XrdEc::AlignedWrite2MissingTest .............. Passed 0.05 sec + Start 16: XrdEc::AlignedWriteTestIsalCrcNoMt +16/23 Test #16: XrdEc::AlignedWriteTestIsalCrcNoMt ........... Passed 0.06 sec + Start 17: XrdEc::SmallWriteTestIsalCrcNoMt +17/23 Test #17: XrdEc::SmallWriteTestIsalCrcNoMt ............. Passed 0.06 sec + Start 18: XrdEc::BigWriteTestIsalCrcNoMt +18/23 Test #18: XrdEc::BigWriteTestIsalCrcNoMt ............... Passed 0.06 sec + Start 19: XrdEc::AlignedWrite1MissingTestIsalCrcNoMt +19/23 Test #19: XrdEc::AlignedWrite1MissingTestIsalCrcNoMt ... Passed 0.06 sec + Start 20: XrdEc::AlignedWrite2MissingTestIsalCrcNoMt +20/23 Test #20: XrdEc::AlignedWrite2MissingTestIsalCrcNoMt ... Passed 0.06 sec + Start 21: XRootD::start +21/23 Test #21: XRootD::start ................................ Passed 0.01 sec + Start 23: XRootD::smoke-test +22/23 Test #23: XRootD::smoke-test ........................... Passed 1.63 sec + Start 22: XRootD::stop +23/23 Test #22: XRootD::stop ................................. Passed 1.00 sec + +100% tests passed, 0 tests failed out of 23 + +Total Test time (real) = 16.55 sec +``` + +For full verbose output, use `-VV` instead of `-V`. We recommend using at least `-V` +to add some verbosity. The output is too terse to be useful otherwise. + +### Customizing the Build + +#### Selecting a build type, compile flags, optional features, etc + +Since the script is targeted for usage with continuous integration, it tries to +load a configuration file from the `.ci` subdirectory in the source directory. +The default configuration is in the `config.cmake` file. This file is used to +pre-load the CMake cache. If found, it is passed to CMake during configuration +via the `-C` option. This file is a CMake script that should only contain CMake +`set()` commands using the `CACHE` option to populate the cache. Some effort is +made to detect and use a more specific configuration file than the generic +`config.cmake` that is used by default. For example, on Ubuntu, a file named +`ubuntu.cmake` will be used if present. The script also tries to detect the +version of the OS and use a more specific file if found for that version. For +example, on Alma Linux 8, one could use `almalinux8.cmake` which would have +higher precedence than `almalinux.cmake`. The default `config.cmake` file will +enable as many options as possible without failing if the dependencies are not +installed, so it should be sufficient in most cases. + +The behavior of the [test.cmake](../test.cmake) script can also be influenced +by environment variables like `CC`, `CXX`, `CXXFLAGS`, `CMAKE_ARGS`, `CMAKE_GENERATOR`, +`CMAKE_BUILD_PARALLEL_LEVEL`, `CTEST_PARALLEL_LEVEL`, and `CTEST_CONFIGURATION_TYPE`. +These are mostly self-explanatory and can be used to override the provided defaults. +For example, to build with `clang` and use `ninja` as CMake generator, one can run: + +```sh +xrootd $ env CC=clang CXX=clang++ CMAKE_GENERATOR=Ninja ctest -V -S test.cmake +``` + +For performance analysis and profiling with `perf`, we recommend building with + +```sh +xrootd $ CXXFLAGS='-fno-omit-frame-pointer' ctest -V -C RelWithDebInfo -S test.cmake +``` + +For enabling link-time optimizations (LTO), we recommend using +``` +CXXFLAGS='-flto -Werror=odr -Werror=lto-type-mismatch -Werror=strict-aliasing' +``` + +This turns some important warnings into errors to avoid potential runtime issues +with LTO. Please see GCC's manual page for descriptions of each of the warnings +above. XRootD also support using address and thread sanitizers, via the options +`-DENABLE_ASAN=ON` and `-DENABLE_TSAN=ON`, respectively. These should be enabled +using `CMAKE_ARGS`, as shown below + +```sh +$ env CMAKE_ARGS="-DENABLE_TSAN=1" ctest -V -S test.cmake +``` + +Note that options passed by setting `CMAKE_ARGS` in the environment have higher +precedence than what is in the pre-loaded cache file, so this method can be used +to override the defaults without having to edit the pre-loaded cache file. + +#### Enabling coverage, memory checking, and static analysis + +The [test.cmake](../test.cmake) has are several options that allow the developer +to customize the build being tested. The main options are shown in the table +below: + +| Option | Description | +|:------------------------:|:-------------------------------------------| +| **-DCOVERAGE=ON** | Enables test coverage analysis with gcov | +| **-DMEMCHECK=ON** | Enables memory checking with valgrind | +| **-DSTATIC_ANALYSIS=ON** | Enables static analysis with clang-tidy | +| **-DINSTALL=ON** | Enables an extra step to call make install | + +When enabling coverage, a report is generated by default in the `html/` +directory inside the build directory. The results can then be viewed by +opening the file `html/coverage_details.html` in a web browser. The report +generation step can be disabled by passing the option `-DGCOVR=0` to the +script. If `gcovr` is not found, the step will be skipped automatically. +It is recommended to use a debug build to generate the coverage analysis. + +The CMake build type can be specified directly on the command line with the +`-C` option of `ctest`. For example, to run a coverage build in debug mode, +showing test output when a test failure happens, one can run: + +```sh +xrootd $ ctest -V --output-on-failure -C Debug -DCOVERAGE=ON -S test.cmake +``` + +Memory checking is performed with `valgrind` when it is enabled. In this case, +the tests are run twice, once as usual, and once with `valgrind`. The output +logs from running the tests with `valgrind` can be found in the build directory +at `build/Testing/Temporary/MemoryChecker.<#>.log` where `<#>` corresponds to +the test number as shown when running `ctest`. + +Static analysis requires `clang-tidy` and is enabled by setting `CMAKE_CXX_CLANG_TIDY` +for the build. If `clang-tidy` is not in the standard `PATH`, then it may be +necessary to set it manually instead of using the option `-DSTATIC_ANALYSIS=ON`. +For the moment XRootD does not provide yet its own configuration file for +`clang-tidy`, so the defaults will be used for the build. Warnings and errors +coming from static analysis will be shown as part of the regular build, so it is +important to enable full verbosity when enabling static analysis to be able to +see the output from `clang-tidy`. + +The option `-DINSTALL=ON` will enable a step to perform a so-called staged +installation inside the build directory. It can be used to check if the +installation procedure is working as expected, by inspecting the contents of the +`install/` directory inside the build directory after installation: + +```sh +xrootd $ ctest -DINSTALL=1 -S test.cmake + Each . represents 1024 bytes of output + ..... Size of output: 4K + Each symbol represents 1024 bytes of output. + '!' represents an error and '*' a warning. + .................................................. Size: 49K + .. Size of output: 52K + Each symbol represents 1024 bytes of output. + '!' represents an error and '*' a warning. + ............................................*!.... Size: 49K + . Size of output: 50K +Error(s) when building project +xrootd $ tree -L 3 -d build/install +build/install +└── usr + ├── bin + ├── include + │   └── xrootd + ├── lib + │   └── python3.11 + ├── lib64 + └── share + ├── man + └── xrootd + +11 directories +``` + +Note that, as shown above, `CTest` erroneously shows build errors when +installing XRootD with this command. This is because of a deprecation +warning emitted by `pip` while installing the Python bindings and can be +safely ignored. + +### Dependencies Required for Coverage, Memory Checking, and Static Analysis + +#### RPM-based distributions: RedHat, Fedora, CentOS, Alma, Rocky + +The [test.cmake](../test.cmake) script may also need some extra dependencies for +some of its features. For memory checking, `valgrind` is needed, and for static +analysis, `clang-tidy` is needed: + +```sh +dnf install \ + clang \ + clang-tools-extra \ + valgrind +``` + +For coverage, you need to install `gcovr`. It is not available via `yum`/`dnf`, +but can be installed with `pip`. See https://gcovr.com/en/stable/installation.html +for more information. + +Dependencies to run containerized tests with `podman` on RHEL 8/9 and derivatives +can be installed with `dnf groupinstall 'Container Management'`. On CentOS 7 and +RHEL 7, one can use `docker` by installing it with `yum install docker`. In this +case, you will need to ensure that your user is in the `docker` group so that +you can run docker without using `sudo`, and that the system daemon for Docker +is running. A quick test to check if everything is correctly setup is to try to +start a busybox image: `docker run --rm -it busybox`. + +#### DEB-based distributions: Debian 11, Ubuntu 22.04 + +On Debian, Ubuntu, and derivatives, The extra dependencies to use with the [test.cmake](../test.cmake) script can be +installed with: + +```sh +apt install clang clang-tidy valgrind gcovr +``` + +Dependencies to run containerized tests can be installed with +```sh +apt install podman +``` + +## Running XRootD Tests on other platforms with Docker and/or Podman + +If you would like to run XRootD tests on other platforms, you can use +the `xrd-docker` script and associated `Dockerfile`s in the `docker/` +subdirectory. The steps needed are described below. + +### Create an XRootD tarball to build in the container + +The first thing that needs to be done is packaging a tarball with the version +of XRootD to be used to build in the container image. The command `xrd-docker package` +by default creates a tarball named `xrootd.tar.gz` in the current directory using the +`HEAD` of the currently checked branch. We recommend changing directory to the `docker/` +directory in the XRootD git repository in order to run these commands. Suppose +we would like to run the tests for release v5.6.4. Then, we would run +```sh +$ xrd-docker package v5.6.4 +``` +to create the tarball that will be used to build the container image. The +tarball created by this command is a standard tarball created with `git archive`. +Inside it, the `VERSION` file contains the expanded version which is used by the +new spec file to detect the version of XRootD being built. You can also create a +source RPM with such tarballs, but they must be built with `rpmbuild --with git` +as done in the CI builds and the `Dockerfile`s in the `docker/build/` subdirectory. + +### Build the container image + +The next step is to build the container image for the desired OS. It can be built +with either `docker` or `podman`. The `xrd-docker` script has the `build` command +to facilitate this. Currently, supported OSs for building are CentOS 7, AlmaLinux 8, +AlmaLinux 9, Fedora. The command to build the image is simply +```sh +$ xrd-docker build +``` +where `` is one of `centos7` (default), `alma8`, `alma9`, or `fedora`. The +name simply chooses which `Dockerfile` is used from the `build/` directory, as +they are named `Dockerfile.` for each suported OS. It is possible to add new +`Dockerfile`s following this same naming scheme to support custom setups and +still use `xrd-docker build` command to build an image. The images built with +`xrd-docker build` are named simply `xrootd` (latest being a default tag added +by docker), and an extra `xrootd:` tag is added to allow having it built for +multiple OSs at the same time. The current `Dockerfile`s use the spec file and +build the image using the RPM packaging workflow, installing dependencies as +declared in the spec file, in the first stage, building the RPMs in a second +stage, then, in a third stage starting from a fresh image, the RPMs built in +stage 2 are copied over and installed with `yum` or `dnf`. + +#### Switching between `docker` and `podman` if both are installed + +The `xrd-docker` script takes either `docker` or `podman` if available, in this +order. If you have only one of the two installed, everything should work without +any extra setup, but if you have both installed and would like to use `podman` +instead of `docker` for building the images, it can be done by exporting an +environment variable: +```sh +$ export DOCKER=$(command -v podman) +$ xrd-docker build # uses podman from now on... +``` + +### Appendix + +#### Setting up `podman` + +Unlike `docker`, `podman` may not work out of the box after installing it. If it +doesn't, make sure that you have subuids and subgids setup for your user by +running, for example, the two commands below: + +```sh +$ sudo usermod --add-subuids 1000000-1000999999 $(id -un) +$ sudo usermod --add-subgids 1000000-1000999999 $(id -un) +``` + +You may also have to ensure that container registries in +`/etc/containers/registries.conf`. Usually a usable configuration can be renamed +from `/etc/containers/registries.conf.example`. + +Finally, you may want to try container runtimes other than the default. If you +still have problems getting started, `podman`'s documentation can be found at +`https://podman.io/docs`. diff --git a/docs/man/CMakeLists.txt b/docs/man/CMakeLists.txt new file mode 100644 index 00000000000..bacd32217f2 --- /dev/null +++ b/docs/man/CMakeLists.txt @@ -0,0 +1,58 @@ +if( XRDCL_LIB_ONLY ) + return() +endif() + +set(MAN1PAGES + xrdcp.1 + xrdfs.1 + xrdmapc.1 +) + +set(MAN8PAGES + cmsd.8 + frm_admin.8 + frm_purged.8 + frm_xfragent.8 + frm_xfrd.8 + mpxstats.8 + xrdpfc_print.8 + xrdpwdadmin.8 + xrdsssadmin.8 + xrootd.8 +) + +if ( BUILD_FUSE ) + list(APPEND MAN1PAGES xrootdfs.1) +endif() + +if ( NOT XRDCL_ONLY ) + list(APPEND MAN1PAGES xrdadler32.1 xrdgsiproxy.1 xrdgsitest.1) + + foreach(MAN ${MAN8PAGES}) + configure_file(${MAN} man8/${MAN} @ONLY) + endforeach() + + install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/man8 + DESTINATION ${CMAKE_INSTALL_MANDIR}) + + if( BUILD_VOMS ) + list(APPEND MAN1PAGES libXrdVoms.1) + endif() +endif() + +foreach(MAN ${MAN1PAGES}) + configure_file(${MAN} man1/${MAN} @ONLY) +endforeach() + +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/man1 + DESTINATION ${CMAKE_INSTALL_MANDIR}) + +install(CODE + "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink xrdcp.1 xrdcopy.1 + WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man1)") + +if( BUILD_VOMS ) + install(CODE + "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink libXrdVoms.1 libXrdSecgsiVOMS.1 + WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man1)") +endif() diff --git a/docs/man/cmsd.8 b/docs/man/cmsd.8 index e1bc1690fac..b271c5906ab 100644 --- a/docs/man/cmsd.8 +++ b/docs/man/cmsd.8 @@ -1,4 +1,4 @@ -.TH cmsd 8 "__VERSION__" +.TH cmsd 8 "@XRootD_VERSION_STRING@" .SH NAME cmsd - Cluster Management Services daemon .SH SYNOPSIS diff --git a/docs/man/frm_admin.8 b/docs/man/frm_admin.8 index ecdf1d165df..6b83cfc6c88 100644 --- a/docs/man/frm_admin.8 +++ b/docs/man/frm_admin.8 @@ -1,4 +1,4 @@ -.TH frm_admin 8 "__VERSION__" +.TH frm_admin 8 "@XRootD_VERSION_STRING@" .SH NAME frm_admin - administer file residency parameters .SH SYNOPSIS diff --git a/docs/man/frm_purged.8 b/docs/man/frm_purged.8 index 72885c00950..e4e020bb5f4 100644 --- a/docs/man/frm_purged.8 +++ b/docs/man/frm_purged.8 @@ -1,4 +1,4 @@ -.TH frm_purged 8 "__VERSION__" +.TH frm_purged 8 "@XRootD_VERSION_STRING@" .SH NAME frm_purged - File Residency Manager purge daemon .SH SYNOPSIS diff --git a/docs/man/frm_xfragent.8 b/docs/man/frm_xfragent.8 index 54fd1062315..c9e6857b25e 100644 --- a/docs/man/frm_xfragent.8 +++ b/docs/man/frm_xfragent.8 @@ -1,4 +1,4 @@ -.TH frm_xfragent 8 "__VERSION__" +.TH frm_xfragent 8 "@XRootD_VERSION_STRING@" .SH NAME frm_xfragent - File Residency Manager Transfer agent .SH SYNOPSIS diff --git a/docs/man/frm_xfrd.8 b/docs/man/frm_xfrd.8 index f550b627589..bc53f170f44 100644 --- a/docs/man/frm_xfrd.8 +++ b/docs/man/frm_xfrd.8 @@ -1,4 +1,4 @@ -.TH frm_xfrd 8 "__VERSION__" +.TH frm_xfrd 8 "@XRootD_VERSION_STRING@" .SH NAME frm_xfrd - File Residency Manager transfer daemon .SH SYNOPSIS diff --git a/docs/man/libXrdVoms.1 b/docs/man/libXrdVoms.1 index cec36f00039..651bb6cc35e 100644 --- a/docs/man/libXrdVoms.1 +++ b/docs/man/libXrdVoms.1 @@ -1,4 +1,4 @@ -.TH libXrdVoms 1 "__VERSION__" +.TH libXrdVoms 1 "@XRootD_VERSION_STRING@" .SH NAME libXrdVoms - XRootD plug-in to extract VOMS attributes .SH SYNOPSIS diff --git a/docs/man/mpxstats.8 b/docs/man/mpxstats.8 index 722fd1412d7..edd17a545a9 100644 --- a/docs/man/mpxstats.8 +++ b/docs/man/mpxstats.8 @@ -1,4 +1,4 @@ -.TH mpxstats 8 "__VERSION__" +.TH mpxstats 8 "@XRootD_VERSION_STRING@" .SH NAME mpxstats - Multiplexing Monitor Statistics daemon .SH SYNOPSIS diff --git a/docs/man/xrdadler32.1 b/docs/man/xrdadler32.1 index 4a778a3ac10..fe31c7e88b5 100644 --- a/docs/man/xrdadler32.1 +++ b/docs/man/xrdadler32.1 @@ -1,4 +1,4 @@ -.TH xrdadler32 1 "__VERSION__" +.TH xrdadler32 1 "@XRootD_VERSION_STRING@" .SH NAME xrdadler32 - compute and display an adler32 checksum .SH SYNOPSIS diff --git a/docs/man/xrdcp.1 b/docs/man/xrdcp.1 index 1e2f0699b3c..126d8ea65ce 100644 --- a/docs/man/xrdcp.1 +++ b/docs/man/xrdcp.1 @@ -1,4 +1,4 @@ -.TH xrdcopy 1 "__VERSION__" +.TH xrdcopy 1 "@XRootD_VERSION_STRING@" .SH NAME xrdcp - copy files .SH SYNOPSIS @@ -332,7 +332,7 @@ The default value is \fIOFF\fR. XRD_PARALLELEVTLOOP .RS 5 -The number of event loops. +The number of event loops (i.e. the number of threads handling requests). Default number is 10. .RE XRD_READRECOVERY diff --git a/docs/man/xrdfs.1 b/docs/man/xrdfs.1 index a84c7e17b09..fd5ddd291c5 100644 --- a/docs/man/xrdfs.1 +++ b/docs/man/xrdfs.1 @@ -1,4 +1,4 @@ -.TH xrdfs 1 "__VERSION__" +.TH xrdfs 1 "@XRootD_VERSION_STRING@" .SH NAME xrdfs - xrootd file and directory meta-data utility .SH SYNOPSIS diff --git a/docs/man/xrdgsiproxy.1 b/docs/man/xrdgsiproxy.1 index ade5272331c..c63517a69ab 100644 --- a/docs/man/xrdgsiproxy.1 +++ b/docs/man/xrdgsiproxy.1 @@ -1,4 +1,4 @@ -.TH xrdgsiproxy 1 "__VERSION__" +.TH xrdgsiproxy 1 "@XRootD_VERSION_STRING@" .SH NAME xrdgsiproxy - generate a proxy X.509 certificate .SH SYNOPSIS diff --git a/docs/man/xrdgsitest.1 b/docs/man/xrdgsitest.1 index ee59e006238..1eca0cfbb6b 100644 --- a/docs/man/xrdgsitest.1 +++ b/docs/man/xrdgsitest.1 @@ -1,4 +1,4 @@ -.TH xrdgsitest 1 "__VERSION__" +.TH xrdgsitest 1 "@XRootD_VERSION_STRING@" .SH NAME xrdgsitest - test crypto functionality relevant for the GSI implementation .SH SYNOPSIS diff --git a/docs/man/xrdmapc.1 b/docs/man/xrdmapc.1 index 9807a487812..7ed05ff9530 100644 --- a/docs/man/xrdmapc.1 +++ b/docs/man/xrdmapc.1 @@ -1,4 +1,4 @@ -.TH xrdmapc 1 "__VERSION__" +.TH xrdmapc 1 "@XRootD_VERSION_STRING@" .SH NAME xrdmapc - query XRootD redirector (status/subscribers/paths) .SH SYNOPSIS diff --git a/docs/man/xrdpfc_print.8 b/docs/man/xrdpfc_print.8 index 948283e3cd0..e8dbf846c3a 100644 --- a/docs/man/xrdpfc_print.8 +++ b/docs/man/xrdpfc_print.8 @@ -1,4 +1,4 @@ -.TH xrdpfc_print 8 "__VERSION__" +.TH xrdpfc_print 8 "@XRootD_VERSION_STRING@" .SH NAME xrdpfc_print - print content of XRootd ProxyFileCache meta data .SH SYNOPSIS diff --git a/docs/man/xrdpwdadmin.8 b/docs/man/xrdpwdadmin.8 index aff35737ab8..2d3903e7dd8 100644 --- a/docs/man/xrdpwdadmin.8 +++ b/docs/man/xrdpwdadmin.8 @@ -1,4 +1,4 @@ -.TH xrdpwdadmin 8 "__VERSION__" +.TH xrdpwdadmin 8 "@XRootD_VERSION_STRING@" .SH NAME xrdpwdadmin - administer pwd security protocol passwords .SH SYNOPSIS diff --git a/docs/man/xrdsssadmin.8 b/docs/man/xrdsssadmin.8 index 235d4ed5636..40bbc13efa5 100644 --- a/docs/man/xrdsssadmin.8 +++ b/docs/man/xrdsssadmin.8 @@ -1,4 +1,4 @@ -.TH xrdsssadmin 8 "__VERSION__" +.TH xrdsssadmin 8 "@XRootD_VERSION_STRING@" .SH NAME xrdsssadmin - administer simple shared secret keytables .SH SYNOPSIS diff --git a/docs/man/xrootd.8 b/docs/man/xrootd.8 index 022318ae98a..d954993b70c 100644 --- a/docs/man/xrootd.8 +++ b/docs/man/xrootd.8 @@ -1,4 +1,4 @@ -.TH xrootd 8 "__VERSION__" +.TH xrootd 8 "@XRootD_VERSION_STRING@" .SH NAME xrootd - eXtended ROOT daemon .SH SYNOPSIS diff --git a/docs/man/xrootdfs.1 b/docs/man/xrootdfs.1 index 92ae7adc69d..11e87865573 100644 --- a/docs/man/xrootdfs.1 +++ b/docs/man/xrootdfs.1 @@ -1,4 +1,4 @@ -.TH xrootdfs 1 "__VERSION__" +.TH xrootdfs 1 "@XRootD_VERSION_STRING@" .SH NAME xrootdfs - xrootd FUSE file system daemon .SH SYNOPSIS diff --git a/gen-tarball.sh b/gen-tarball.sh index aa1605e6ed5..cb05ec7c2ac 100755 --- a/gen-tarball.sh +++ b/gen-tarball.sh @@ -1,39 +1,11 @@ #!/bin/bash -# Generate a source tarball including submodules -if [ -z "${1}" ] ; then - echo No tag or branch given - exit 1 -fi -ver=${1} -# Remove initial v from tag name for use in filenames -if [ ${ver:0:1} = 'v' ] ; then - fver=${ver:1} -else - fver=${ver} -fi -if [ -r xrootd-${fver}.tar.gz ] ; then - echo xrootd-${fver}.tar.gz already exists - exit 1 -fi -curdir=$(pwd) -tdir=$(mktemp -d) -cd ${tdir} -git clone https://github.com/xrootd/xrootd.git -cd xrootd -git checkout ${ver} -if [ $? -ne 0 ] ; then - echo No such tag or branch: ${ver} - cd ${curdir} - rm -rf ${tdir} - exit 1 -fi -git archive --prefix xrootd-${fver}/ ${ver} -o ${tdir}/xrootd-${fver}.tar -git submodule update --init -git submodule foreach --recursive "git archive --prefix xrootd-${fver}/\$path/ \$sha1 -o ${tdir}/\$sha1.tar ; tar -A -f ${tdir}/xrootd-${fver}.tar ${tdir}/\$sha1.tar ; rm ${tdir}/\$sha1.tar" -cd ${tdir} -gzip xrootd-${fver}.tar -mv xrootd-${fver}.tar.gz ${curdir} -cd ${curdir} -rm -rf ${tdir} +set -e + +TAG=$(printf "%s" "$(git describe "${1:-HEAD}")") +NAME="xrootd-${TAG#v}" + +set -x + +git archive -9 --prefix="${NAME}/" -o "${NAME}.tar.gz" ${TAG} diff --git a/genversion.sh b/genversion.sh index e0c6f7f7200..39723b89dc8 100755 --- a/genversion.sh +++ b/genversion.sh @@ -1,244 +1,63 @@ #!/usr/bin/env bash -#------------------------------------------------------------------------------- -# Process the git decoration expansion and try to derive version number -#------------------------------------------------------------------------------- -EXP1='^v[12][0-9][0-9][0-9][01][0-9][0-3][0-9]-[0-2][0-9][0-5][0-9]$' -EXP2='^v[0-9]+\.[0-9]+\.[0-9]+$' -EXP3='^v[0-9]+\.[0-9]+\.[0-9]+\-rc.*$' - -#------------------------------------------------------------------------------- -# Get the numeric version -#------------------------------------------------------------------------------- -function getNumericVersion() -{ - VERSION=$1 - if test x`echo $VERSION | grep -E $EXP2` == x; then - echo "1000000"; - return; - fi - VERSION=${VERSION/v/} - VERSION=${VERSION//./ } - VERSION=($VERSION) - printf "%d%02d%02d\n" ${VERSION[0]} ${VERSION[1]} ${VERSION[2]} -} - -#------------------------------------------------------------------------------- -# Extract version number from git references -#------------------------------------------------------------------------------- -function getVersionFromRefs() -{ - REFS=${1/RefNames:/} - REFS=${REFS//,/} - REFS=${REFS/(/} - REFS=${REFS/)/} - REFS=($REFS) - - VERSION="unknown" - - for i in ${REFS[@]}; do - if test x`echo $i | grep -E $EXP2` != x; then - echo "$i" - return 0 - fi - - if test x`echo $i | grep -E $EXP1` != x; then - VERSION="$i" - fi - - if test x`echo $i | grep -E $EXP3` != x; then - VERSION="$i" - fi - - done - echo $VERSION - return 0 -} - -#------------------------------------------------------------------------------- -# Generate the version string from the date and the hash -#------------------------------------------------------------------------------- -function getVersionFromLog() -{ - AWK=gawk - EX="$(command -v gawk)" - if test x"${EX}" == x -o ! -x "${EX}"; then - AWK=awk - fi - - VERSION="`echo $@ | $AWK '{ gsub("-","",$1); print $1"-"$4; }'`" - if test $? -ne 0; then - echo "unknown"; - return 1 - fi - echo v$VERSION -} - -#------------------------------------------------------------------------------- -# Print help -#------------------------------------------------------------------------------- -function printHelp() +# This script no longer generates XrdVersion.hh, but just +# prints the version using the same strategy as in the module +# XRootDVersion.cmake. The script will first try to use a custom +# version set with the option --version or via the environment +# variable XRDVERSION, then read the VERSION file and if that is +# not expanded by git, git describe is used. If a bad version is +# set for any reason, a fallback version is used based on a date. + +function usage() { - echo "Usage:" 1>&2 - echo "${0} [--help|--print-only|--version] [SOURCEPATH]" 1>&2 - echo " --help prints this message" 1>&2 - echo " --print-only prints the version to stdout and quits" 1>&2 - echo " --version VERSION sets the version manually" 1>&2 + cat 1>&2 <<-EOF + Usage: + $(basename $0) [--help|--version] + + --help prints this message + --print-only ignored, used for backward compatibility + --version VERSION sets a custom version + EOF } -#------------------------------------------------------------------------------- -# Check the parameters -#------------------------------------------------------------------------------- -while test ${#} -ne 0; do - if test x${1} = x--help; then - PRINTHELP=1 - elif test x${1} = x--print-only; then - PRINTONLY=1 - elif test x${1} = x--version; then - if test ${#} -lt 2; then - echo "--version parameter needs an argument" 1>&2 - exit 1 - fi - USER_VERSION=${2} - shift - else - SOURCEPATH=${1} - fi - shift -done +SRC=$(dirname $0) +VF=${SRC}/VERSION -if test x$PRINTHELP != x; then - printHelp ${0} - exit 0 -fi - -if test x$SOURCEPATH != x; then - SOURCEPATH=${SOURCEPATH}/ - if test ! -d $SOURCEPATH; then - echo "The given source path does not exist: ${SOURCEPATH}" 1>&2 - exit 1 - fi -fi - -VERSION="unknown" - -#------------------------------------------------------------------------------- -# We're not inside a git repo -#------------------------------------------------------------------------------- -if test ! -d ${SOURCEPATH}.git; then - #----------------------------------------------------------------------------- - # We cannot figure out what version we are - #---------------------------------------------------------------------------- - echo "[I] No git repository info found. Trying to interpret VERSION_INFO" 1>&2 - if test -f src/XrdVersion.hh; then - echo "[I] The XrdVersion.hh file already exists" 1>&2 - exit 0 - elif test ! -r ${SOURCEPATH}VERSION_INFO; then - echo "[!] VERSION_INFO file absent. Unable to determine the version. Using \"unknown\"" 1>&2 - elif test x"`grep Format ${SOURCEPATH}VERSION_INFO`" != x; then - echo "[!] VERSION_INFO file invalid. Unable to determine the version. Using \"unknown\"" 1>&2 - elif test x$USER_VERSION != x; then - echo "[I] Using the user supplied version: ${USER_VERSION}" 1>&2 - VERSION=${USER_VERSION} - #----------------------------------------------------------------------------- - # The version file exists and seems to be valid so we know the version - #---------------------------------------------------------------------------- - else - REFNAMES="`grep RefNames ${SOURCEPATH}VERSION_INFO`" - VERSION="`getVersionFromRefs "$REFNAMES"`" - if test x$VERSION == xunknown; then - SHORTHASH="`grep ShortHash ${SOURCEPATH}VERSION_INFO`" - SHORTHASH=${SHORTHASH/ShortHash:/} - SHORTHASH=${SHORTHASH// /} - DATE="`grep Date ${SOURCEPATH}VERSION_INFO`" - DATE=${DATE/Date:/} - VERSION="`getVersionFromLog $DATE $SHORTHASH`" - fi - fi - -#------------------------------------------------------------------------------- -# Check if the version has been specified by the user -#------------------------------------------------------------------------------- -elif test x$USER_VERSION != x; then - VERSION=$USER_VERSION - -#------------------------------------------------------------------------------- -# We're in a git repo so we can try to determine the version using that -#------------------------------------------------------------------------------- +if [[ -n "${XRDVERSION}" ]]; then + VERSION=${XRDVERSION}; +elif [[ -r "${VF}" ]] && grep -vq "Format:" "${VF}"; then + VERSION=$(sed -e 's/-g/+g/' "${VF}") +elif git -C ${SRC} describe >/dev/null 2>&1; then + VERSION=$(git -C ${SRC} describe | sed -e 's/-g/+g/') else - echo "[I] Determining version from git" 1>&2 - EX="$(command -v git)" - if test x"${EX}" == x -o ! -x "${EX}"; then - echo "[!] Unable to find git in the path: setting the version tag to unknown" 1>&2 - else - #--------------------------------------------------------------------------- - # Sanity check - #--------------------------------------------------------------------------- - CURRENTDIR=$PWD - if [ x${SOURCEPATH} != x ]; then - cd ${SOURCEPATH} - fi - git log -1 >/dev/null 2>&1 - if test $? -ne 0; then - echo "[!] Error while generating src/XrdVersion.hh, the git repository may be corrupted" 1>&2 - echo "[!] Setting the version tag to unknown" 1>&2 - else - #------------------------------------------------------------------------- - # Can we match the exact annotated tag? - #------------------------------------------------------------------------- - git describe --abbrev=0 --exact-match >/dev/null 2>&1 - - if test ${?} -eq 0; then - VERSION=$(git describe --abbrev=0 --exact-match) - else - VERSION=$(git describe --abbrev=0) - # Append .postN with N equal to number of commits since last tag - VERSION="${VERSION}.post$(git rev-list ${VERSION}.. | wc -l)" - fi - fi - cd $CURRENTDIR - fi -fi - -#------------------------------------------------------------------------------- -# Make sure the version string is not longer than 25 characters -#------------------------------------------------------------------------------- -if [ ${#VERSION} -gt 25 ] && [ x$USER_VERSION == x ] ; then - VERSION="${VERSION:0:19}...${VERSION: -3}" -fi - -#------------------------------------------------------------------------------- -# Print the version info and exit if necassary -#------------------------------------------------------------------------------- -if test x$PRINTONLY != x; then - echo $VERSION - exit 0 -fi - -#------------------------------------------------------------------------------- -# Create XrdVersion.hh -#------------------------------------------------------------------------------- -NUMVERSION=`getNumericVersion $VERSION` - -if test ! -r ${SOURCEPATH}src/XrdVersion.hh.in; then - echo "[!] Unable to find src/XrdVersion.hh.in" 1>&2 - exit 1 -fi - -sed -e "s/#define XrdVERSION \"unknown\"/#define XrdVERSION \"$VERSION\"/" ${SOURCEPATH}src/XrdVersion.hh.in | \ -sed -e "s/#define XrdVNUMBER 1000000/#define XrdVNUMBER $NUMVERSION/" \ -> src/XrdVersion.hh.new - -if test $? -ne 0; then - echo "[!] Error while generating src/XrdVersion.hh from the input template" 1>&2 - exit 1 -fi + VERSION="v5.7-rc$(date +%Y%m%d)" +fi + +while [[ $# -gt 0 ]]; do + case $1 in + --help) + usage + exit 0 + ;; + + --print-only) + shift + ;; + + --version) + shift + if [[ $# == 0 ]]; then + echo "error: --version parameter needs an argument" 1>&2 + fi + VERSION=$1 + shift + ;; + + *) + echo "warning: unknown option: $1" 1>&2 + shift + ;; + esac +done -if test ! -e src/XrdVersion.hh; then - mv src/XrdVersion.hh.new src/XrdVersion.hh -elif test x"`diff src/XrdVersion.hh.new src/XrdVersion.hh`" != x; then - mv src/XrdVersion.hh.new src/XrdVersion.hh -else - rm src/XrdVersion.hh.new -fi -echo "[I] src/XrdVersion.hh successfully generated" 1>&2 +printf "${VERSION}" diff --git a/packaging/common/cmsd@.service b/packaging/common/cmsd@.service index 76d2d6b679b..f69e0e54d5d 100644 --- a/packaging/common/cmsd@.service +++ b/packaging/common/cmsd@.service @@ -6,6 +6,14 @@ Requires=network-online.target After=network-online.target [Service] +#PrivateDevices=true +#ProtectHostname=true +#ProtectClock=true +#ProtectKernelTunables=true +#ProtectKernelModules=true +#ProtectKernelLogs=true +#ProtectControlGroups=true +#RestrictRealtime=true ExecStart=/usr/bin/cmsd -l /var/log/xrootd/cmsd.log -c /etc/xrootd/xrootd-%i.cfg -k fifo -s /run/xrootd/cmsd-%i.pid -n %i User=xrootd Group=xrootd diff --git a/packaging/common/frm_purged@.service b/packaging/common/frm_purged@.service index 375e41aff3a..942dbf5adb2 100644 --- a/packaging/common/frm_purged@.service +++ b/packaging/common/frm_purged@.service @@ -6,6 +6,14 @@ Requires=network-online.target After=network-online.target [Service] +#PrivateDevices=true +#ProtectHostname=true +#ProtectClock=true +#ProtectKernelTunables=true +#ProtectKernelModules=true +#ProtectKernelLogs=true +#ProtectControlGroups=true +#RestrictRealtime=true ExecStart=/usr/bin/frm_purged -l /var/log/xrootd/frm_purged.log -c /etc/xrootd/xrootd-%i.cfg -k fifo -s /run/xrootd/frm_purged-%i.pid -n %i User=xrootd Group=xrootd diff --git a/packaging/common/frm_xfrd@.service b/packaging/common/frm_xfrd@.service index 0106ac061b5..cfc580db2f2 100644 --- a/packaging/common/frm_xfrd@.service +++ b/packaging/common/frm_xfrd@.service @@ -6,6 +6,14 @@ Requires=network-online.target After=network-online.target [Service] +#PrivateDevices=true +#ProtectHostname=true +#ProtectClock=true +#ProtectKernelTunables=true +#ProtectKernelModules=true +#ProtectKernelLogs=true +#ProtectControlGroups=true +#RestrictRealtime=true ExecStart=/usr/bin/frm_xfrd -l /var/log/xrootd/frm_xfrd.log -c /etc/xrootd/xrootd-%i.cfg -k fifo -s /run/xrootd/frm_xfrd-%i.pid -n %i User=xrootd Group=xrootd diff --git a/packaging/common/xrootd@.service b/packaging/common/xrootd@.service index c8c1279893e..1c8284c9c89 100644 --- a/packaging/common/xrootd@.service +++ b/packaging/common/xrootd@.service @@ -6,6 +6,14 @@ Requires=network-online.target After=network-online.target [Service] +#PrivateDevices=true +#ProtectHostname=true +#ProtectClock=true +#ProtectKernelTunables=true +#ProtectKernelModules=true +#ProtectKernelLogs=true +#ProtectControlGroups=true +#RestrictRealtime=true ExecStart=/usr/bin/xrootd -l /var/log/xrootd/xrootd.log -c /etc/xrootd/xrootd-%i.cfg -k fifo -s /run/xrootd/xrootd-%i.pid -n %i User=xrootd Group=xrootd diff --git a/packaging/debian/control b/packaging/debian/control deleted file mode 100644 index 42f1ac24d64..00000000000 --- a/packaging/debian/control +++ /dev/null @@ -1,395 +0,0 @@ -Source: xrootd -Maintainer: Jozsef Makai -Section: net -Priority: optional -Standards-Version: 3.9.3 -Build-Depends: debhelper (>= 9), cmake (>=3.3.0), zlib1g-dev, libfuse-dev, python3-dev, libssl-dev, libxml2-dev, ncurses-dev, libkrb5-dev, libreadline-dev, libsystemd-dev, selinux-policy-dev, libcurl4-openssl-dev, systemd, uuid-dev, dh-python, voms-dev, davix-dev, python3-pip, python3-setuptools, pkgconf -Homepage: https://github.com/xrootd/xrootd -Vcs-Git: https://github.com/xrootd/xrootd.git -Vcs-Browser: https://github.com/xrootd/xrootd - -#------------------------------------------------------------------------------ -# XRootD libraries (compatibility) -#------------------------------------------------------------------------------ -Package: xrootd-libs -Architecture: any -Section: libs -Depends: libxrdapputils1 (=${binary:Version}), - libxrdcrypto1 (=${binary:Version}), - libxrdcryptolite1 (=${binary:Version}), - libxrdutils2 (=${binary:Version}), - libxrdxml2 (=${binary:Version}), - xrootd-plugins (=${binary:Version}) -Conflicts: xrootd-libs (<<${binary:Version}) -Description: This package contains libraries used by the xrootd servers and - clients. - . - This is a transitional dummy package which can be removed - afterinstallation of dependencies. - -#------------------------------------------------------------------------------ -# XRootD plug-ins -#------------------------------------------------------------------------------ -Package: xrootd-plugins -Architecture: any -Section: net -Depends: ${shlibs:Depends} -Conflicts: xrootd-plugins (<<${binary:Version}) -Description: This package contains additional plugins used by both client and - server. - -#------------------------------------------------------------------------------ -# XRootD development files (compatibility) -#------------------------------------------------------------------------------ -Package: xrootd-devel -Architecture: any -Section: libdevel -Depends: libxrootd-dev (=${binary:Version}) -Conflicts: xrootd-devel (<<${binary:Version}) -Description: This package contains header files and development libraries for - xrootd development. - . - This is a transitional dummy package which can be removed after - installation of dependencies - -#------------------------------------------------------------------------------ -# XRootD development files -#------------------------------------------------------------------------------ -Package: libxrootd-dev -Architecture: any -Section: libdevel -Depends: libxrdapputils1 (=${binary:Version}), - libxrdcrypto1 (=${binary:Version}), - libxrdcryptolite1 (=${binary:Version}), - libxrdutils2 (=${binary:Version}), - libxrdxml2 (=${binary:Version}), -Conflicts: xrootd-devel (<<${binary:Version}) -Description: This package contains header files and development libraries for - xrootd development. - -#------------------------------------------------------------------------------ -# XRootD client libraries (compatibility) -#------------------------------------------------------------------------------ -Package: xrootd-client-libs -Architecture: any -Depends: libxrdcl2 (=${binary:Version}), - libxrdffs2 (=${binary:Version}), - libxrdposix2 (=${binary:Version}), - xrootd-client-plugins (=${binary:Version}), - xrootd-libs (=${binary:Version}) -Conflicts: xrootd-client-libs (<<${binary:Version}) -Description: This package contains libraries used by xrootd clients. - . - This is a transitional dummy package which can be removed after - installation of dependencies - -#------------------------------------------------------------------------------ -# XRootD client plug-ins -#------------------------------------------------------------------------------ -Package: xrootd-client-plugins -Architecture: any -Section: net -Depends: ${shlibs:Depends} -Conflicts: xrootd-client-plugins (<<${binary:Version}) -Description: This package contains additional plugins used by the client. - -#------------------------------------------------------------------------------ -# XRootD client development files (compatibility) -#------------------------------------------------------------------------------ -Package: xrootd-client-devel -Architecture: any -Section: libdevel -Depends: libxrootd-client-dev (=${binary:Version}) -Conflicts: xrootd-client-devel (<<${binary:Version}) -Description: This package contains header files and development libraries for - xrootd client development. - . - This is a transitional dummy package which can be removed after - installation of dependencies - -#------------------------------------------------------------------------------ -# XRootD client development files -#------------------------------------------------------------------------------ -Package: libxrootd-client-dev -Architecture: any -Section: libdevel -Depends: ${shlibs:Depends}, - libxrootd-dev (=${binary:Version}), - libxrdcl2 (=${binary:Version}), - libxrdffs2 (=${binary:Version}), - libxrdposix2 (=${binary:Version}) -Conflicts: xrootd-client-devel (<<${binary:Version}) -Description: This package contains header files and development libraries for - xrootd client development. - -#------------------------------------------------------------------------------ -# XRootD client (compatibility) -#------------------------------------------------------------------------------ -Package: xrootd-client -Architecture: any -Section: net -Depends: xrootd-clients (=${binary:Version}) -Conflicts: xrootd-client (<<${binary:Version}) -Description: This package contains the command line tools used to communicate - with xrootd servers. - . - This is a transitional dummy package which can be removed after - installation of dependencies - -#------------------------------------------------------------------------------ -# XRootD client -#------------------------------------------------------------------------------ -Package: xrootd-clients -Architecture: any -Section: net -Depends: ${shlibs:Depends}, - xrootd-client-libs (=${binary:Version}) -Conflicts: xrootd-clients (<<${binary:Version}) -Description: This package contains the command line tools used to communicate - with xrootd servers. - -#------------------------------------------------------------------------------ -# XRootD server -#------------------------------------------------------------------------------ -Package: xrootd-server -Architecture: any -Section: net -Depends: ${shlibs:Depends}, - xrootd-server-libs (=${binary:Version}) -Conflicts: xrootd-server (<<${binary:Version}) -Description: This package contains the server applications and associated - utilities. - -#------------------------------------------------------------------------------ -# XRootD private development files (compatibility) -#------------------------------------------------------------------------------ -Package: xrootd-private-devel -Architecture: any -Section: libdevel -Depends: libxrootd-private-dev (=${binary:Version}) -Conflicts: xrootd-private-devel (<<${binary:Version}) -Description: This package contains some private xrootd headers. The use of these - headers is strongly discouraged. Backward compatibility between - versions is not guaranteed for these headers. - . - This is a transitional dummy package which can be removed after - installation of dependencies - -#------------------------------------------------------------------------------ -# XRootD private development files -#------------------------------------------------------------------------------ -Package: libxrootd-private-dev -Architecture: any -Section: libdevel -Depends: libxrootd-server-dev (=${binary:Version}), -Conflicts: libxrootd-private-dev (<<${binary:Version}) -Description: This package contains some private XRootd headers. The use of - these headers is strongly discouraged. Backward compatibility - between versions is not guaranteed for these headers. - -#------------------------------------------------------------------------------ -# XRootD server libs (compatibility) -#------------------------------------------------------------------------------ -Package: xrootd-server-libs -Architecture: any -Section: libs -Depends: libxrdhttputils1 (=${binary:Version}), - libxrdserver2 (=${binary:Version}), - libxrdssilib1 (=${binary:Version}), - libxrdssishmap1 (=${binary:Version}), - xrootd-client-libs (=${binary:Version}), - xrootd-server-plugins (=${binary:Version}) -Conflicts: xrootd-server-libs (<<${binary:Version}) -Description: This package contains libraries used by xrootd servers. - . - This is a transitional dummy package which can be removed after - installation of dependencies - -#------------------------------------------------------------------------------ -# XRootD server development files (compatibility) -#------------------------------------------------------------------------------ -Package: xrootd-server-devel -Architecture: any -Section: libdevel -Depends: libxrootd-server-dev (=${binary:Version}) -Conflicts: xrootd-server-devel (<<${binary:Version}) -Description: This package contains header files and development libraries for - xrootd server development. - . - This is a transitional dummy package which can be removed after - installation of dependencies - -#------------------------------------------------------------------------------ -# XRootD server development files -#------------------------------------------------------------------------------ -Package: libxrootd-server-dev -Architecture: any -Section: libdevel -Depends: libxrdhttputils1 (=${binary:Version}), - libxrdserver2 (=${binary:Version}), - libxrdssilib1 (=${binary:Version}), - libxrdssishmap1 (=${binary:Version}), - libxrootd-client-dev (=${binary:Version}) -Conflicts: libxrootd-server-dev (<<${binary:Version}) -Description: High performance, scalable fault tolerant data repositories. - This package contains header files and development libraries for - xrootd server development. - -#------------------------------------------------------------------------------ -# XRootD server development files -#------------------------------------------------------------------------------ -Package: xrootd-fuse -Architecture: any -Section: net -Depends: ${shlibs:Depends}, - libfuse-dev, - xrootd-client-libs (=${binary:Version}), -Conflicts: xrootd-fuse (<<${binary:Version}) -Description: This package contains the FUSE (file system in user space) - xrootd mount tool. - -#------------------------------------------------------------------------------ -# XRootD server plug-ins -#------------------------------------------------------------------------------ -Package: xrootd-server-plugins -Architecture: any -Section: net -Depends: ${shlibs:Depends} -Description: This package contains additional plug-in libraries used by an - XRootd server. - -#------------------------------------------------------------------------------ -# XRootD libxrdapputils1 (xroottd-libs) -#------------------------------------------------------------------------------ -Package: libxrdapputils1 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdapputils1 (<<${binary:Version}) -Description: Library of utilities for applications. - -#------------------------------------------------------------------------------ -# XRootD libxrdcl2 (xrootd-client-libs) -#------------------------------------------------------------------------------ -Package: libxrdcl2 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdcl2 (<<${binary:Version}) -Description: Client library. - -#------------------------------------------------------------------------------ -# XRootD libxrdcrypto1 (xrootd-libs) -#------------------------------------------------------------------------------ -Package: libxrdcrypto1 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdcrypto1 (<<${binary:Version}) -Description: Cryptograpic library. - -#------------------------------------------------------------------------------ -# XRootD libxrdcryptolite1 (xrootd-libs) -#------------------------------------------------------------------------------ -Package: libxrdcryptolite1 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdcryptolite1 (<<${binary:Version}) -Description: Light version of cryptograpic library. - -#------------------------------------------------------------------------------ -# XRootD libxrdffs2 (xrootd-client-libs) -#------------------------------------------------------------------------------ -Package: libxrdffs2 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdffs2 (<<${binary:Version}) -Description: File protocol library - -#------------------------------------------------------------------------------ -# XRootD libxrdhttputils1 (xrootd-server-libs) -#------------------------------------------------------------------------------ -Package: libxrdhttputils1 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdhttputils1 (<<${binary:Version}) -Description: Library of utilities for HTTP protocol - -#------------------------------------------------------------------------------ -# XRootD libxrdposix2 (xrootd-client-libs) -#------------------------------------------------------------------------------ -Package: libxrdposix2 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdposix2 (<<${binary:Version}) -Description: Posix interface library. - -#------------------------------------------------------------------------------ -# XRootD libxrdserver2 (xrootd-server-libs) -#------------------------------------------------------------------------------ -Package: libxrdserver2 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdserver2 (<<${binary:Version}) -Description: Server library. - -#------------------------------------------------------------------------------ -# XRootD libxrdssilib1 (xrootd-server-libs) -#------------------------------------------------------------------------------ -Package: libxrdssilib1 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdssilib1 (<<${binary:Version}) -Description: Server internals library. - -#------------------------------------------------------------------------------ -# XRootD libxrdssishmap1 (xrootd-server-libs) -#------------------------------------------------------------------------------ -Package: libxrdssishmap1 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdssishmap1 (<<${binary:Version}) -Description: Server internals library - -#------------------------------------------------------------------------------ -# XRootD libxrdutils2 (xrootd-libs) -#------------------------------------------------------------------------------ -Package: libxrdutils2 -Architecture: any -Section: libs -Depends: ${shlibs:Depends} -Conflicts: libxrdutils2 (<<${binary:Version}) -Description: Library of utilities - -#------------------------------------------------------------------------------ -# XRootD libxrdxml2 (xrootd-libs) -#------------------------------------------------------------------------------ -Package: libxrdxml2 -Architecture: any -Section: libs -Depends: ${shlibs:Depends}, - libxml2 -Conflicts: libxrdxml2 (<<${binary:Version}) -Description: XML library - -#------------------------------------------------------------------------------ -# XRootD python3-xrootd -#------------------------------------------------------------------------------ -Package: python3-xrootd -Architecture: any -Section: python -Depends: ${python3:Depends}, - ${shlibs:Depends}, - xrootd-client-libs (=${binary:Version}) -Conflicts: python3-xrootd (<<${binary:Version}) -Provides: ${python3:Provides} -Description: Python interface - - diff --git a/packaging/debian/copyright b/packaging/debian/copyright deleted file mode 100644 index 341c4ec5bbd..00000000000 --- a/packaging/debian/copyright +++ /dev/null @@ -1,26 +0,0 @@ -Copyright (c) 2005-2012, Board of Trustees of the Leland Stanford, -Jr. University. Produced under contract DE-AC02-76-SF00515 with the -US Department of Energy. All rights reserved. The copyright holder's -institutional names may not be used to endorse or promote products -derived from this software without specific prior "written permission. - -This file is part of the XRootD software suite. - -XRootD is free software: you can redistribute it and/or modify it -under the terms of the GNU Lesser General Public License as published -by the Free Software "Foundation, either version 3 of the License, or -(at your option) any later version. - -XRootD is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public -License for more details (/usr/share/common-licenses/LGPL). - -You should have received a copy of the GNU Lesser General Public -License along with XRootD in a file called COPYING.LESSER (LGPL -license) and file COPYING (GPL license). If not, see -. - -Prior to September 2nd, 2012 the XRootD software suite was licensed -under a modified BSD license (see file COPYING.BSD). This applies to -all code that was in the XRootD git repository prior to that date. diff --git a/packaging/debian/python3-xrootd.install b/packaging/debian/python3-xrootd.install deleted file mode 100644 index b882c0b561d..00000000000 --- a/packaging/debian/python3-xrootd.install +++ /dev/null @@ -1 +0,0 @@ -usr/lib/python3*/site-packages/* diff --git a/packaging/debian/python3-xrootd.install.new b/packaging/debian/python3-xrootd.install.new deleted file mode 100644 index 6c1d7e1948e..00000000000 --- a/packaging/debian/python3-xrootd.install.new +++ /dev/null @@ -1 +0,0 @@ -usr/local/lib/python*/dist-packages/* \ No newline at end of file diff --git a/packaging/debian/xrootd-client-plugins.install b/packaging/debian/xrootd-client-plugins.install deleted file mode 100644 index c7b320992a6..00000000000 --- a/packaging/debian/xrootd-client-plugins.install +++ /dev/null @@ -1,4 +0,0 @@ -usr/lib/*/libXrdClProxyPlugin-5.so -usr/lib/*/libXrdPosixPreload.so* -usr/lib/*/libXrdClHttp-5.so -etc/xrootd/client.plugins.d/xrdcl-http-plugin.conf \ No newline at end of file diff --git a/packaging/debian/xrootd-client.install b/packaging/debian/xrootd-client.install deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packaging/debian/xrootd-fuse.install b/packaging/debian/xrootd-fuse.install deleted file mode 100644 index 4353d0fed3a..00000000000 --- a/packaging/debian/xrootd-fuse.install +++ /dev/null @@ -1,3 +0,0 @@ -usr/bin/xrootdfs -usr/share/man/man1/xrootdfs.1* -etc/xrootd \ No newline at end of file diff --git a/packaging/debian/xrootd-plugins.install b/packaging/debian/xrootd-plugins.install deleted file mode 100644 index b4fa550bc4e..00000000000 --- a/packaging/debian/xrootd-plugins.install +++ /dev/null @@ -1,3 +0,0 @@ -usr/lib/*/libXrdCks*-5.so -usr/lib/*/libXrdCryptossl-5.so -usr/lib/*/libXrdSec*-5.so diff --git a/packaging/debian/xrootd-server-plugins.install b/packaging/debian/xrootd-server-plugins.install deleted file mode 100644 index d66668eff39..00000000000 --- a/packaging/debian/xrootd-server-plugins.install +++ /dev/null @@ -1,13 +0,0 @@ -usr/lib/*/libXrdBwm-5.so -usr/lib/*/libXrdPss-5.so -usr/lib/*/libXrdXrootd-5.so -usr/lib/*/libXrdPfc-5.so -usr/lib/*/libXrdBlacklistDecision-5.so -usr/lib/*/libXrdHttp-5.so -usr/lib/*/libXrdHttpTPC-5.so -usr/lib/*/libXrdN2No2p-5.so -usr/lib/*/libXrdOssSIgpfsT-5.so -usr/lib/*/libXrdSsi-5.so -usr/lib/*/libXrdSsiLog-5.so -usr/lib/*/libXrdThrottle-5.so -usr/lib/*/libXrdCmsRedirectLocal-5.so diff --git a/packaging/debian/xrootd-server.install b/packaging/debian/xrootd-server.install deleted file mode 100644 index 3442b30314e..00000000000 --- a/packaging/debian/xrootd-server.install +++ /dev/null @@ -1,23 +0,0 @@ -usr/bin/cconfig -usr/bin/cmsd -usr/bin/frm_admin -usr/bin/frm_purged -usr/bin/frm_xfragent -usr/bin/frm_xfrd -usr/bin/mpxstats -usr/bin/wait41 -usr/bin/xrdacctest -usr/bin/xrdpfc_print -usr/bin/xrdpwdadmin -usr/bin/xrdsssadmin -usr/bin/xrootd -usr/share/man/man8/cmsd.8 -usr/share/man/man8/frm_admin.8 -usr/share/man/man8/frm_purged.8 -usr/share/man/man8/frm_xfragent.8 -usr/share/man/man8/frm_xfrd.8 -usr/share/man/man8/mpxstats.8 -usr/share/man/man8/xrdpfc_print.8 -usr/share/man/man8/xrdpwdadmin.8 -usr/share/man/man8/xrdsssadmin.8 -usr/share/man/man8/xrootd.8 diff --git a/packaging/makesrpm.sh b/packaging/makesrpm.sh index 817ffdd2f1f..f3af058f0e1 100755 --- a/packaging/makesrpm.sh +++ b/packaging/makesrpm.sh @@ -4,22 +4,6 @@ # Author: Lukasz Janyst (10.03.2011) #------------------------------------------------------------------------------- -RCEXP='^[0-9]+\.[0-9]+\.[0-9]+\-rc.*$' -CERNEXP='^[0-9]+\.[0-9]+\.[0-9]+\-[0-9]+\.CERN.*$' - -#------------------------------------------------------------------------------- -# Find a program -#------------------------------------------------------------------------------- -function findProg() -{ - for prog in $@; do - if test -x "$(command -v $prog 2>/dev/null)"; then - echo $prog - break - fi - done -} - #------------------------------------------------------------------------------- # Print help #------------------------------------------------------------------------------- @@ -39,7 +23,7 @@ function printHelp() #------------------------------------------------------------------------------- # Parse the commandline, if only we could use getopt... :( #------------------------------------------------------------------------------- -SOURCEPATH="../" +SOURCEPATH=$(realpath $(dirname $0)/..) OUTPUTPATH="." PRINTHELP=0 @@ -65,7 +49,7 @@ while test ${#} -ne 0; do echo "--version parameter needs an argument" 1>&2 exit 1 fi - USER_VERSION="--version ${2}" + VERSION="${2}" shift elif test x${1} = x--define; then if test ${#} -lt 2; then @@ -108,12 +92,12 @@ fi #------------------------------------------------------------------------------- # Check if we have all the necassary components #------------------------------------------------------------------------------- -if test x`findProg rpmbuild` = x; then +if ! command -v rpmbuild 2>/dev/null; then echo "[!] Unable to find rpmbuild, aborting..." 1>&2 exit 1 fi -if test x`findProg git` = x; then +if ! command -v git 2>/dev/null; then echo "[!] Unable to find git, aborting..." 1>&2 exit 1 fi @@ -126,15 +110,9 @@ if test ! -d $SOURCEPATH/.git; then exit 2 fi -#------------------------------------------------------------------------------- -# Check the version number -#------------------------------------------------------------------------------- -if test ! -x $SOURCEPATH/genversion.sh; then - echo "[!] Unable to find the genversion script" 1>&2 - exit 3 -fi +: ${VERSION:=$(git --git-dir=$SOURCEPATH/.git describe)} +: ${RELEASE:=1} -VERSION=`$SOURCEPATH/genversion.sh --print-only $USER_VERSION $SOURCEPATH 2>/dev/null` if test $? -ne 0; then echo "[!] Unable to figure out the version number" 1>&2 exit 4 @@ -142,39 +120,17 @@ fi echo "[i] Working with version: $VERSION" -if test x${VERSION:0:1} = x"v"; then - VERSION=${VERSION:1} -fi - #------------------------------------------------------------------------------- -# Deal with release candidates +# Sanitize version to work with RPMs +# https://docs.fedoraproject.org/en-US/packaging-guidelines/Versioning/ #------------------------------------------------------------------------------- -RELEASE=1 -if test x`echo $VERSION | grep -E $RCEXP` != x; then - RELEASE=0.`echo $VERSION | sed 's/.*-rc/rc/'` - VERSION=`echo $VERSION | sed 's/-rc.*//'` -fi -#------------------------------------------------------------------------------- -# Deal with CERN releases -#------------------------------------------------------------------------------- -if test x`echo $VERSION | grep -E $CERNEXP` != x; then - RELEASE=`echo $VERSION | sed 's/.*-//'` - VERSION=`echo $VERSION | sed 's/-.*\.CERN//'` -fi +VERSION=${VERSION#v} # remove "v" prefix +VERSION=${VERSION/-rc/~rc} # release candidates use ~ in RPMs +VERSION=${VERSION/-g*/} # snapshots versions not supported well, filter out +VERSION=${VERSION/-/.post} # handle git describe for post releases +VERSION=${VERSION//-/.} # replace remaining dashes with dots -#------------------------------------------------------------------------------- -# In case of user version check if the release number has been provided -#------------------------------------------------------------------------------- -if test x"$USER_VERSION" != x; then - TMP=`echo $VERSION | sed 's#.*-##g'` - if test $TMP != $VERSION; then - RELEASE=$TMP - VERSION=`echo $VERSION | sed 's#-[^-]*$##'` - fi -fi - -VERSION=`echo $VERSION | sed 's/-/./g'` echo "[i] RPM compliant version: $VERSION-$RELEASE" #------------------------------------------------------------------------------- @@ -239,39 +195,6 @@ if test $? -ne 0; then exit 6 fi -#------------------------------------------------------------------------------- -# Make sure submodules are in place -#------------------------------------------------------------------------------- -git submodule init -git submodule update -- src/XrdClHttp -git submodule update -- src/XrdCeph -#git submodule foreach git pull origin master - -#------------------------------------------------------------------------------- -# Add XrdCeph sub-module to our tarball -#------------------------------------------------------------------------------- -cd src/XrdCeph - -if [ -z ${TAG+x} ]; then - COMMIT=`git log --pretty=format:"%H" -1` -else - COMMIT=$TAG -fi - -git archive --prefix=xrootd/src/XrdCeph/ --format=tar $COMMIT > $RPMSOURCES/xrootd-ceph.tar -if test $? -ne 0; then - echo "[!] Unable to create the xrootd-ceph source tarball" 1>&2 - exit 6 -fi - -tar --concatenate --file $RPMSOURCES/xrootd.tar $RPMSOURCES/xrootd-ceph.tar -if test $? -ne 0; then - echo "[!] Unable to add xrootd-ceph to xrootd tarball" 1>&2 - exit 6 -fi - -cd - > /dev/null - #------------------------------------------------------------------------------- # gzip the tarball #------------------------------------------------------------------------------- diff --git a/packaging/rhel/xrootd.spec.in b/packaging/rhel/xrootd.spec.in deleted file mode 100644 index 3ddc5b01fe4..00000000000 --- a/packaging/rhel/xrootd.spec.in +++ /dev/null @@ -1,1212 +0,0 @@ -#------------------------------------------------------------------------------- -# Helper macros -#------------------------------------------------------------------------------- -%if %{?rhel:1}%{!?rhel:0} - # starting with rhel 7 we have systemd and macaroons, - %define use_systemd 1 - %define have_macaroons 1 - - %if %{rhel} == 7 - # we build both python2 and python3 bindings for EPEL7 - %define _with_python2 1 - %define _with_python3 1 - %else - # we only build both python3 bindings for EPEL>7 - %define _with_python2 0 - %define _with_python3 1 - %endif -%else - # do we have macaroons ? - %if %{?fedora}%{!?fedora:0} >= 28 - %define have_macaroons 1 - %else - %define have_macaroons 0 - %endif - # do we have systemd ? - %if %{?fedora}%{!?fedora:0} >= 19 - %define use_systemd 1 - %else - %define use_systemd 0 - %endif - # we only build python3 bindings for fedora - %define _with_python2 0 - %define _with_python3 1 -%endif - - - -%if %{?_with_ceph11:1}%{!?_with_ceph11:0} - %define _with_ceph 1 -%endif - -%if %{?rhel:1}%{!?rhel:0} - %if %{rhel} > 7 - %define use_cmake3 0 - %else - %define use_cmake3 1 - %endif -%else - %define use_cmake3 0 -%endif - - -# Remove default rpm python bytecompiling scripts -%global __os_install_post \ - %(echo '%{__os_install_post}' | \ - sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g \ - s!/usr/lib[^[:space:]]*/brp-python-hardlink[[:space:]].*$!!g') - -#------------------------------------------------------------------------------- -# Package definitions -#------------------------------------------------------------------------------- -Name: xrootd -Epoch: 1 -Version: __VERSION__ -Release: __RELEASE__%{?dist}%{?_with_clang:.clang}%{?_with_asan:.asan} -Summary: Extended ROOT file server -Group: System Environment/Daemons -License: LGPLv3+ -URL: http://xrootd.org/ - -%define compat_version 4.12.3 - -# git clone http://xrootd.org/repo/xrootd.git xrootd -# cd xrootd -# git-archive master | gzip -9 > ~/rpmbuild/SOURCES/xrootd.tgz -Source0: xrootd.tar.gz - -%if 0%{?_with_compat} -Source1: xrootd-%{compat_version}.tar.gz -%endif - -BuildRoot: %{_tmppath}/%{name}-root - -%if %{use_cmake3} -BuildRequires: cmake3 -%else -BuildRequires: cmake -%endif -BuildRequires: krb5-devel -BuildRequires: readline-devel -BuildRequires: fuse-devel -BuildRequires: libxml2-devel -BuildRequires: krb5-devel -BuildRequires: zlib-devel -BuildRequires: ncurses-devel -BuildRequires: libcurl-devel -BuildRequires: libuuid-devel -BuildRequires: voms-devel >= 2.0.6 -BuildRequires: git -BuildRequires: pkgconfig -%if %{have_macaroons} -BuildRequires: libmacaroons-devel -%endif -BuildRequires: json-c-devel - -%if %{_with_python2} -BuildRequires: python2-pip -BuildRequires: python2-devel -BuildRequires: python2-setuptools -%endif -%if %{_with_python3} -BuildRequires: python%{python3_pkgversion}-devel -BuildRequires: python%{python3_pkgversion}-setuptools -%endif - -BuildRequires: openssl-devel - -BuildRequires: selinux-policy-devel - -%if %{?_with_tests:1}%{!?_with_tests:0} -BuildRequires: cppunit-devel -BuildRequires: gtest-devel -%endif - -%if %{?_with_ceph:1}%{!?_with_ceph:0} - %if %{?_with_ceph11:1}%{!?_with_ceph11:0} -BuildRequires: librados-devel >= 11.0 -BuildRequires: libradosstriper-devel >= 11.0 - %else -BuildRequires: ceph-devel >= 0.87 - %endif -%endif - -%if %{?_with_xrdclhttp:1}%{!?_with_xrdclhttp:0} -BuildRequires: davix-devel -%endif - -BuildRequires: doxygen -BuildRequires: graphviz -%if %{?rhel}%{!?rhel:0} == 5 -BuildRequires: graphviz-gd -%endif - -%if %{?_with_clang:1}%{!?_with_clang:0} -BuildRequires: clang -%endif - -%if %{?_with_asan:1}%{!?_with_asan:0} -BuildRequires: libasan -%if %{?rhel}%{!?rhel:0} == 7 -BuildRequires: devtoolset-7-libasan-devel -%endif -Requires: libasan -%endif - -%if %{?_with_scitokens:1}%{!?_with_scitokens:0} -BuildRequires: scitokens-cpp-devel -%endif - -%if %{?_with_isal:1}%{!?_with_isal:0} -BuildRequires: autoconf -BuildRequires: automake -BuildRequires: libtool -BuildRequires: yasm -%endif - -Requires: %{name}-server%{?_isa} = %{epoch}:%{version}-%{release} -Requires: %{name}-selinux = %{epoch}:%{version}-%{release} - -%if %{use_systemd} -BuildRequires: systemd -BuildRequires: systemd-devel -Requires(pre): systemd -Requires(post): systemd -Requires(preun): systemd -Requires(postun): systemd -%else -Requires(pre): shadow-utils -Requires(pre): chkconfig -Requires(post): chkconfig -Requires(preun): chkconfig -Requires(preun): initscripts -Requires(postun): initscripts -%endif - -%if %{?rhel}%{!?rhel:0} == 7 -BuildRequires: devtoolset-7 -%else -BuildRequires: gcc-c++ -%endif - -%description -The Extended root file server consists of a file server called xrootd -and a cluster management server called cmsd. - -The xrootd server was developed for the root analysis framework to -serve root files. However, the server is agnostic to file types and -provides POSIX-like access to any type of file. - -The cmsd server is the next generation version of the olbd server, -originally developed to cluster and load balance Objectivity/DB AMS -database servers. It provides enhanced capability along with lower -latency and increased throughput. - -#------------------------------------------------------------------------------- -# libs -#------------------------------------------------------------------------------- -%package libs -Summary: Libraries used by xrootd servers and clients -Group: System Environment/Libraries - -%description libs -This package contains libraries used by the xrootd servers and clients. - -#------------------------------------------------------------------------------- -# devel -#------------------------------------------------------------------------------ -%package devel -Summary: Development files for xrootd -Group: Development/Libraries -Requires: %{name}-libs%{?_isa} = %{epoch}:%{version}-%{release} - -%description devel -This package contains header files and development libraries for xrootd -development. - -#------------------------------------------------------------------------------- -# client-libs -#------------------------------------------------------------------------------- -%package client-libs -Summary: Libraries used by xrootd clients -Group: System Environment/Libraries -Requires: %{name}-libs%{?_isa} = %{epoch}:%{version}-%{release} - -%description client-libs -This package contains libraries used by xrootd clients. - -#------------------------------------------------------------------------------- -# client-devel -#------------------------------------------------------------------------------- -%package client-devel -Summary: Development files for xrootd clients -Group: Development/Libraries -Requires: %{name}-devel%{?_isa} = %{epoch}:%{version}-%{release} -Requires: %{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release} - -%description client-devel -This package contains header files and development libraries for xrootd -client development. - -#------------------------------------------------------------------------------- -# server-libs -#------------------------------------------------------------------------------- -%package server-libs -Summary: Libraries used by xrootd servers -Group: System Environment/Libraries -Requires: %{name}-libs%{?_isa} = %{epoch}:%{version}-%{release} -Requires: %{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release} -Obsoletes: xrootd-macaroons -Obsoletes: xrootd-tpc - -%description server-libs -This package contains libraries used by xrootd servers. - -#------------------------------------------------------------------------------- -# server-devel -#------------------------------------------------------------------------------- -%package server-devel -Summary: Development files for xrootd servers -Group: Development/Libraries -Requires: %{name}-devel%{?_isa} = %{epoch}:%{version}-%{release} -Requires: %{name}-client-devel%{?_isa} = %{epoch}:%{version}-%{release} -Requires: %{name}-server-libs%{?_isa} = %{epoch}:%{version}-%{release} - -%description server-devel -This package contains header files and development libraries for xrootd -server development. - -#------------------------------------------------------------------------------- -# private devel -#------------------------------------------------------------------------------- -%package private-devel -Summary: Private xrootd headers -Group: Development/Libraries -Requires: %{name}-devel%{?_isa} = %{epoch}:%{version}-%{release} -Requires: %{name}-client-devel%{?_isa} = %{epoch}:%{version}-%{release} -Requires: %{name}-server-devel%{?_isa} = %{epoch}:%{version}-%{release} - -%description private-devel -This package contains some private xrootd headers. Backward and forward -compatibility between versions is not guaranteed for these headers. - -#------------------------------------------------------------------------------- -# client -#------------------------------------------------------------------------------- -%package client -Summary: Xrootd command line client tools -Group: Applications/Internet -Requires: %{name}-libs%{?_isa} = %{epoch}:%{version}-%{release} -Requires: %{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release} - -%description client -This package contains the command line tools used to communicate with -xrootd servers. - -#------------------------------------------------------------------------------- -# server -#------------------------------------------------------------------------------- -%package server -Summary: Extended ROOT file server -Group: System Environment/Daemons -Requires: %{name}-libs = %{epoch}:%{version}-%{release} -Requires: %{name}-client-libs = %{epoch}:%{version}-%{release} -Requires: %{name}-server-libs = %{epoch}:%{version}-%{release} -Requires: expect - -%description server -XRootD server binaries - -#------------------------------------------------------------------------------- -# fuse -#------------------------------------------------------------------------------- -%package fuse -Summary: Xrootd FUSE tool -Group: Applications/Internet -Requires: %{name}-libs%{?_isa} = %{epoch}:%{version}-%{release} -Requires: %{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release} -Requires: fuse - -%description fuse -This package contains the FUSE (file system in user space) xrootd mount -tool. - -#------------------------------------------------------------------------------- -# Python bindings -#------------------------------------------------------------------------------- - -%if %{_with_python2} -#------------------------------------------------------------------------------- -# python2 -#------------------------------------------------------------------------------- -%package -n python2-%{name} -Summary: Python 2 bindings for XRootD -Group: Development/Libraries -Provides: python-%{name} -Provides: %{name}-python = %{epoch}:%{version}-%{release} -Obsoletes: %{name}-python < 1:4.8.0-1 -Requires: %{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release} - -%description -n python2-xrootd -Python 2 bindings for XRootD -%endif - -%if %{_with_python3} -#------------------------------------------------------------------------------- -# python3 -#------------------------------------------------------------------------------- -%package -n python%{python3_pkgversion}-%{name} -Summary: Python 3 bindings for XRootD -Group: Development/Libraries -%{?python_provide:%python_provide python%{python3_pkgversion}-%{name}} -Requires: %{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release} - -%description -n python%{python3_pkgversion}-%{name} -Python 3 bindings for XRootD -%endif - -#------------------------------------------------------------------------------- -# doc -#------------------------------------------------------------------------------- -%package doc -Summary: Developer documentation for the xrootd libraries -Group: Documentation -%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6 -BuildArch: noarch -%endif - -%description doc -This package contains the API documentation of the xrootd libraries. - -#------------------------------------------------------------------------------- -# selinux -#------------------------------------------------------------------------------- -%package selinux -Summary: SELinux policy extensions for xrootd. -Group: System Environment/Base -%if %{?fedora}%{!?fedora:0} >= 10 || %{?rhel}%{!?rhel:0} >= 6 -BuildArch: noarch -%endif -Requires(post): policycoreutils -Requires(postun): policycoreutils -Requires: selinux-policy - -%description selinux -SELinux policy extensions for running xrootd while in enforcing mode. - -#------------------------------------------------------------------------------- -# ceph -#------------------------------------------------------------------------------- -%if %{?_with_ceph:1}%{!?_with_ceph:0} -%package ceph -Summary: Ceph back-end plug-in for XRootD -Group: Development/Tools -Requires: %{name}-server = %{epoch}:%{version}-%{release} -%description ceph -Ceph back-end plug-in for XRootD. -%endif - -#------------------------------------------------------------------------------- -# xrdcl-http -#------------------------------------------------------------------------------- -%if %{?_with_xrdclhttp:1}%{!?_with_xrdclhttp:0} -%package -n xrdcl-http -Summary: HTTP client plug-in for XRootD client -Group: System Environment/Libraries -Requires: %{name}-client = %{epoch}:%{version}-%{release} -%description -n xrdcl-http -xrdcl-http is an XRootD client plugin which allows XRootD to interact -with HTTP repositories. -%endif - -#------------------------------------------------------------------------------- -# xrootd-voms -#------------------------------------------------------------------------------- -%package voms -Summary: VOMS attribute extractor plug-in for XRootD -Group: System Environment/Libraries -Provides: vomsxrd = %{epoch}:%{version}-%{release} -Obsoletes: vomsxrd < 1:4.12.4-1 -Requires: %{name}-libs = %{epoch}:%{version}-%{release} -Obsoletes: xrootd-voms-plugin -%description voms -The VOMS attribute extractor plug-in for XRootD. - -#------------------------------------------------------------------------------- -# xrootd-scitokens -#------------------------------------------------------------------------------- -%if %{?_with_scitokens:1}%{!?_with_scitokens:0} -%package scitokens -Summary: SciTokens authentication plugin for XRootD -Group: Development/Tools -Requires: %{name}-server = %{epoch}:%{version}-%{release} -%description scitokens -SciToken athorization plug-in for XRootD. -%endif - -#------------------------------------------------------------------------------- -# tests -#------------------------------------------------------------------------------- -%if %{?_with_tests:1}%{!?_with_tests:0} -%package tests -Summary: CPPUnit tests -Group: Development/Tools -Requires: %{name}-client = %{epoch}:%{version}-%{release} -%description tests -This package contains a set of CPPUnit tests for xrootd. -%endif - -%if 0%{?_with_compat} -#------------------------------------------------------------------------------- -# client-compat -#------------------------------------------------------------------------------- -%package client-compat -Summary: XRootD 4 compatibility client libraries -Group: System Environment/Libraries - -%description client-compat -This package contains compatibility libraries for xrootd 4 clients. - -#------------------------------------------------------------------------------- -# server-compat -#------------------------------------------------------------------------------- -%package server-compat -Summary: XRootD 4 compatibility server binaries -Group: System Environment/Daemons -Requires: %{name}-libs%{?_isa} = %{epoch}:%{version}-%{release} - -%description server-compat -This package contains compatibility binaries for xrootd 4 servers. - -%endif - -#------------------------------------------------------------------------------- -# Build instructions -#------------------------------------------------------------------------------- -%prep -%if 0%{?_with_compat} -%setup -c -n xrootd-compat -a 1 -T -%endif - -%setup -c -n xrootd - -%build - -%if %{?rhel}%{!?rhel:0} == 7 -. /opt/rh/devtoolset-7/enable -%endif - -cd xrootd - -%if %{?_with_clang:1}%{!?_with_clang:0} -export CC=clang -export CXX=clang++ -%endif - -mkdir build -pushd build - -%if %{use_cmake3} -cmake3 \ -%else -cmake \ -%endif - -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DFORCE_WERROR=TRUE \ -%if %{?_with_tests:1}%{!?_with_tests:0} - -DENABLE_TESTS=TRUE \ -%else - -DENABLE_TESTS=FALSE \ -%endif -%if %{?_with_asan:1}%{!?_with_asan:0} - -DENABLE_ASAN=TRUE \ -%endif -%if %{?_with_ceph:1}%{!?_with_ceph:0} - -DXRDCEPH_SUBMODULE=TRUE \ -%endif -%if %{?_with_xrdclhttp:1}%{!?_with_xrdclhttp:0} - -DENABLE_XRDCLHTTP=TRUE \ -%endif -%if %{?_with_isal:1}%{!?_with_isal:0} - -DENABLE_XRDEC=TRUE \ -%endif - -DUSER_VERSION=v%{version} \ - ../ - -make -i VERBOSE=1 %{?_smp_mflags} -popd - -pushd packaging/common -make -f /usr/share/selinux/devel/Makefile -popd - -doxygen Doxyfile - -%if 0%{?_with_compat} -pushd $RPM_BUILD_DIR/xrootd-compat/xrootd -mkdir build -pushd build -%if %{use_cmake3} -cmake3 \ -%else -cmake \ -%endif - -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DFORCE_WERROR=TRUE \ -%if %{?_with_tests:1}%{!?_with_tests:0} - -DENABLE_TESTS=TRUE \ -%else - -DENABLE_TESTS=FALSE \ -%endif -%if %{?_with_ceph:1}%{!?_with_ceph:0} - -DXRDCEPH_SUBMODULE=TRUE \ -%endif -%if %{?_with_xrdclhttp:1}%{!?_with_xrdclhttp:0} - -DENABLE_XRDEC=TRUE \ -%endif - ../ - -make -i VERBOSE=1 %{?_smp_mflags} -popd -popd -%endif - -pushd build/bindings/python -# build python3 bindings -%if %{_with_python2} -%py2_build -%endif -# build python2 bindings -%if %{_with_python3} -%py3_build -%endif -popd - -%check -cd xrootd/build -%if %{use_cmake3} -ctest3 --output-on-failure -%else -ctest --output-on-failure -%endif - -#------------------------------------------------------------------------------- -# Installation -#------------------------------------------------------------------------------- -%install -rm -rf $RPM_BUILD_ROOT - -#------------------------------------------------------------------------------- -# Install compat -#------------------------------------------------------------------------------- -%if 0%{?_with_compat} -pushd $RPM_BUILD_DIR/xrootd-compat/xrootd/build -make install DESTDIR=$RPM_BUILD_ROOT -rm -rf $RPM_BUILD_ROOT%{_includedir} -rm -rf $RPM_BUILD_ROOT%{_datadir} -rm -f $RPM_BUILD_ROOT%{_bindir}/{cconfig,cns_ssi,frm_admin,frm_xfragent,mpxstats} -rm -f $RPM_BUILD_ROOT%{_bindir}/{wait41,xprep,xrd,xrdadler32,xrdcrc32c,XrdCnsd,xrdcopy} -rm -f $RPM_BUILD_ROOT%{_bindir}/{xrdcks,xrdcp,xrdcp-old,xrdfs,xrdgsiproxy,xrdpwdadmin} -rm -f $RPM_BUILD_ROOT%{_bindir}/{xrdqstats,xrdsssadmin,xrdstagetool,xrootdfs} -rm -f $RPM_BUILD_ROOT%{_libdir}/libXrdAppUtils.so -rm -f $RPM_BUILD_ROOT%{_libdir}/{libXrdClient.so,libXrdCl.so,libXrdCryptoLite.so} -rm -f $RPM_BUILD_ROOT%{_libdir}/{libXrdCrypto.so,libXrdFfs.so,libXrdMain.so} -rm -f $RPM_BUILD_ROOT%{_libdir}/{libXrdOfs.so,libXrdPosixPreload.so,libXrdPosix.so} -rm -f $RPM_BUILD_ROOT%{_libdir}/{libXrdServer.so,libXrdUtils.so} - -for i in cmsd frm_purged frm_xfrd xrootd; do - mv $RPM_BUILD_ROOT%{_bindir}/$i $RPM_BUILD_ROOT%{_bindir}/${i}-4 -done - -rm -f $RPM_BUILD_ROOT%{python2_sitearch}/xrootd-v%{compat_version}*.egg-info -popd -%endif - -#------------------------------------------------------------------------------- -# Install 5.x.y -#------------------------------------------------------------------------------- -pushd xrootd -pushd build -make install DESTDIR=$RPM_BUILD_ROOT -popd - -# configuration stuff -rm -rf $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/* - -# ceph posix unversioned so -rm -f $RPM_BUILD_ROOT%{_libdir}/libXrdCephPosix.so - -# config paths -mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/%{name}/config.d/ - -# var paths -mkdir -p $RPM_BUILD_ROOT%{_var}/log/xrootd -mkdir -p $RPM_BUILD_ROOT%{_var}/run/xrootd -mkdir -p $RPM_BUILD_ROOT%{_var}/spool/xrootd - -# init stuff -mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/xrootd - -%if %{use_systemd} - -mkdir -p $RPM_BUILD_ROOT%{_unitdir} -install -m 644 packaging/common/xrootd@.service $RPM_BUILD_ROOT%{_unitdir} -install -m 644 packaging/common/xrdhttp@.socket $RPM_BUILD_ROOT%{_unitdir} -install -m 644 packaging/common/xrootd@.socket $RPM_BUILD_ROOT%{_unitdir} -install -m 644 packaging/common/cmsd@.service $RPM_BUILD_ROOT%{_unitdir} -install -m 644 packaging/common/frm_xfrd@.service $RPM_BUILD_ROOT%{_unitdir} -install -m 644 packaging/common/frm_purged@.service $RPM_BUILD_ROOT%{_unitdir} - -# tmpfiles.d -mkdir -p $RPM_BUILD_ROOT%{_tmpfilesdir} -install -m 0644 packaging/rhel/xrootd.tmpfiles $RPM_BUILD_ROOT%{_tmpfilesdir}/%{name}.conf - -%else - -mkdir -p $RPM_BUILD_ROOT%{_initrddir} -mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig -install -m 644 packaging/rhel/xrootd.sysconfig $RPM_BUILD_ROOT%{_sysconfdir}/sysconfig/xrootd - -install -m 755 packaging/rhel/cmsd.init $RPM_BUILD_ROOT%{_initrddir}/cmsd -install -m 755 packaging/rhel/frm_purged.init $RPM_BUILD_ROOT%{_initrddir}/frm_purged -install -m 755 packaging/rhel/frm_xfrd.init $RPM_BUILD_ROOT%{_initrddir}/frm_xfrd -install -m 755 packaging/rhel/xrootd.init $RPM_BUILD_ROOT%{_initrddir}/xrootd -install -m 755 packaging/rhel/xrootd.functions $RPM_BUILD_ROOT%{_initrddir}/xrootd.functions - -%endif - -# logrotate -mkdir $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d -install -p -m 644 packaging/common/xrootd.logrotate $RPM_BUILD_ROOT%{_sysconfdir}/logrotate.d/xrootd - -install -m 644 packaging/common/xrootd-clustered.cfg $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/xrootd-clustered.cfg -install -m 644 packaging/common/xrootd-standalone.cfg $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/xrootd-standalone.cfg -install -m 644 packaging/common/xrootd-filecache-clustered.cfg $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/xrootd-filecache-clustered.cfg -install -m 644 packaging/common/xrootd-filecache-standalone.cfg $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/xrootd-filecache-standalone.cfg -%if %{use_systemd} -install -m 644 packaging/common/xrootd-http.cfg $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/xrootd-http.cfg -%endif - -# client plug-in config -mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/client.plugins.d -install -m 644 packaging/common/client-plugin.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/client.plugins.d/client-plugin.conf.example -install -m 644 packaging/common/recorder.conf $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/client.plugins.d/recorder.conf - -%if %{?_with_xrdclhttp:1}%{!?_with_xrdclhttp:0} -install -m 644 packaging/common/http.client.conf.example $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/client.plugins.d/xrdcl-http-plugin.conf -%endif - -# client config -install -m 644 packaging/common/client.conf $RPM_BUILD_ROOT%{_sysconfdir}/xrootd/client.conf - -# documentation -mkdir -p %{buildroot}%{_docdir}/%{name}-%{version} -cp -pr doxydoc/html %{buildroot}%{_docdir}/%{name}-%{version} - -# selinux -mkdir -p %{buildroot}%{_datadir}/selinux/packages/%{name} -install -m 644 -p packaging/common/xrootd.pp \ - %{buildroot}%{_datadir}/selinux/packages/%{name}/%{name}.pp - -pushd build/bindings/python -# install python2 bindings -%if %{_with_python2} -%py2_install -%endif -# install python3 bindings -%if %{_with_python3} -%py3_install -%endif -popd - -%clean -rm -rf $RPM_BUILD_ROOT - -#------------------------------------------------------------------------------- -# RPM scripts -#------------------------------------------------------------------------------- -%post libs -p /sbin/ldconfig -%postun libs -p /sbin/ldconfig - -%post client-libs -p /sbin/ldconfig -%postun client-libs -p /sbin/ldconfig - -%post server-libs -p /sbin/ldconfig -%postun server-libs -p /sbin/ldconfig - -%pre server - -getent group xrootd >/dev/null || groupadd -r xrootd -getent passwd xrootd >/dev/null || \ - useradd -r -g xrootd -c "XRootD runtime user" \ - -s /sbin/nologin -d %{_localstatedir}/spool/xrootd xrootd -exit 0 - -%if %{use_systemd} - -%post server -if [ $1 -eq 1 ] ; then - /usr/bin/systemctl daemon-reload >/dev/null 2>&1 || : -fi - -%preun server -if [ $1 -eq 0 ] ; then - for DAEMON in xrootd cmsd frm_purged frm_xfrd; do - for INSTANCE in `/usr/bin/systemctl | grep $DAEMON@ | awk '{print $1;}'`; do - /usr/bin/systemctl --no-reload disable $INSTANCE > /dev/null 2>&1 || : - /usr/bin/systemctl stop $INSTANCE > /dev/null 2>&1 || : - done - done -fi - -%postun server -if [ $1 -ge 1 ] ; then - /usr/bin/systemctl daemon-reload >/dev/null 2>&1 || : - for DAEMON in xrootd cmsd frm_purged frm_xfrd; do - for INSTANCE in `/usr/bin/systemctl | grep $DAEMON@ | awk '{print $1;}'`; do - /usr/bin/systemctl try-restart $INSTANCE >/dev/null 2>&1 || : - done - done -fi - -%else - -%post server -if [ $1 -eq 1 ]; then - /sbin/chkconfig --add xrootd - /sbin/chkconfig --add cmsd - /sbin/chkconfig --add frm_purged - /sbin/chkconfig --add frm_xfrd -fi - -%preun server -if [ $1 -eq 0 ]; then - /sbin/service xrootd stop >/dev/null 2>&1 || : - /sbin/service cmsd stop >/dev/null 2>&1 || : - /sbin/service frm_purged stop >/dev/null 2>&1 || : - /sbin/service frm_xfrd stop >/dev/null 2>&1 || : - /sbin/chkconfig --del xrootd - /sbin/chkconfig --del cmsd - /sbin/chkconfig --del frm_purged - /sbin/chkconfig --del frm_xfrd -fi - -%postun server -if [ $1 -ge 1 ]; then - /sbin/service xrootd condrestart >/dev/null 2>&1 || : - /sbin/service cmsd condrestart >/dev/null 2>&1 || : - /sbin/service frm_purged condrestart >/dev/null 2>&1 || : - /sbin/service frm_xfrd condrestart >/dev/null 2>&1 || : -fi - -%endif - -#------------------------------------------------------------------------------- -# Add a new user and group if necessary -#------------------------------------------------------------------------------- -%pre fuse -getent group xrootd >/dev/null || groupadd -r xrootd -getent passwd xrootd >/dev/null || \ - useradd -r -g xrootd -c "XRootD runtime user" \ - -s /sbin/nologin -d %{_localstatedir}/spool/xrootd xrootd -exit 0 - -#------------------------------------------------------------------------------- -# Selinux -#------------------------------------------------------------------------------- -%post selinux -/usr/sbin/semodule -i %{_datadir}/selinux/packages/%{name}/%{name}.pp >/dev/null 2>&1 || : - -%postun selinux -if [ $1 -eq 0 ] ; then - /usr/sbin/semodule -r %{name} >/dev/null 2>&1 || : -fi - -#------------------------------------------------------------------------------- -# Files -#------------------------------------------------------------------------------- -%files -# empty - -%files server -%defattr(-,root,root,-) -%{_bindir}/cconfig -%{_bindir}/cmsd -%{_bindir}/frm_admin -%{_bindir}/frm_purged -%{_bindir}/frm_xfragent -%{_bindir}/frm_xfrd -%{_bindir}/mpxstats -%{_bindir}/wait41 -%{_bindir}/xrdpwdadmin -%{_bindir}/xrdsssadmin -%{_bindir}/xrootd -%{_bindir}/xrdpfc_print -%{_bindir}/xrdacctest -%{_mandir}/man8/cmsd.8* -%{_mandir}/man8/frm_admin.8* -%{_mandir}/man8/frm_purged.8* -%{_mandir}/man8/frm_xfragent.8* -%{_mandir}/man8/frm_xfrd.8* -%{_mandir}/man8/mpxstats.8* -%{_mandir}/man8/xrdpwdadmin.8* -%{_mandir}/man8/xrdsssadmin.8* -%{_mandir}/man8/xrootd.8* -%{_mandir}/man8/xrdpfc_print.8* -%{_datadir}/xrootd/utils -%attr(-,xrootd,xrootd) %config(noreplace) %{_sysconfdir}/xrootd/xrootd-clustered.cfg -%attr(-,xrootd,xrootd) %config(noreplace) %{_sysconfdir}/xrootd/xrootd-standalone.cfg -%attr(-,xrootd,xrootd) %config(noreplace) %{_sysconfdir}/xrootd/xrootd-filecache-clustered.cfg -%attr(-,xrootd,xrootd) %config(noreplace) %{_sysconfdir}/xrootd/xrootd-filecache-standalone.cfg -%if %{use_systemd} -%attr(-,xrootd,xrootd) %config(noreplace) %{_sysconfdir}/xrootd/xrootd-http.cfg -%endif -%attr(-,xrootd,xrootd) %dir %{_var}/log/xrootd -%attr(-,xrootd,xrootd) %dir %{_var}/run/xrootd -%attr(-,xrootd,xrootd) %dir %{_var}/spool/xrootd -%attr(-,xrootd,xrootd) %dir %{_sysconfdir}/%{name}/config.d -%config(noreplace) %{_sysconfdir}/logrotate.d/xrootd - -%if %{use_systemd} -%{_unitdir}/* -%{_tmpfilesdir}/%{name}.conf -%else -%config(noreplace) %{_sysconfdir}/sysconfig/xrootd -%{_initrddir}/* -%endif - -%files libs -%defattr(-,root,root,-) -%{_libdir}/libXrdAppUtils.so.2* -%{_libdir}/libXrdClProxyPlugin-5.so -%{_libdir}/libXrdCks*-5.so -%{_libdir}/libXrdCrypto.so.2* -%{_libdir}/libXrdCryptoLite.so.2* -%{_libdir}/libXrdCryptossl-5.so -%{_libdir}/libXrdSec-5.so -%{_libdir}/libXrdSecProt-5.so -%{_libdir}/libXrdSecgsi-5.so -%{_libdir}/libXrdSecgsiAUTHZVO-5.so -%{_libdir}/libXrdSecgsiGMAPDN-5.so -%{_libdir}/libXrdSeckrb5-5.so -%{_libdir}/libXrdSecpwd-5.so -%{_libdir}/libXrdSecsss-5.so -%{_libdir}/libXrdSecunix-5.so -%{_libdir}/libXrdSecztn-5.so -%{_libdir}/libXrdUtils.so.3* -%{_libdir}/libXrdXml.so.3* - -%files devel -%defattr(-,root,root,-) -%dir %{_includedir}/xrootd -%{_bindir}/xrootd-config -%{_includedir}/xrootd/XProtocol -%{_includedir}/xrootd/Xrd -%{_includedir}/xrootd/XrdCks -%{_includedir}/xrootd/XrdNet -%{_includedir}/xrootd/XrdOuc -%{_includedir}/xrootd/XrdSec -%{_includedir}/xrootd/XrdSys -%{_includedir}/xrootd/XrdVersion.hh -%{_libdir}/libXrdAppUtils.so -%{_libdir}/libXrdCrypto.so -%{_libdir}/libXrdCryptoLite.so -%{_libdir}/libXrdUtils.so -%{_libdir}/libXrdXml.so -%{_includedir}/xrootd/XrdXml/XrdXmlReader.hh -%{_datadir}/xrootd/cmake - -%files client-libs -%defattr(-,root,root,-) -%{_libdir}/libXrdCl.so.3* -%{_libdir}/libXrdFfs.so.3* -%{_libdir}/libXrdPosix.so.3* -%{_libdir}/libXrdPosixPreload.so.2* -%{_libdir}/libXrdSsiLib.so.2* -%{_libdir}/libXrdSsiShMap.so.2* -%{_libdir}/libXrdClRecorder-5.so -%if %{?_with_isal:1}%{!?_with_isal:0} -%{_libdir}/libXrdEc.so.1* -%endif -%{_sysconfdir}/xrootd/client.plugins.d/client-plugin.conf.example -%{_sysconfdir}/xrootd/client.plugins.d/recorder.conf -%config(noreplace) %{_sysconfdir}/xrootd/client.conf -# This lib may be used for LD_PRELOAD so the .so link needs to be included -%{_libdir}/libXrdPosixPreload.so - -%files client-devel -%defattr(-,root,root,-) -%{_bindir}/xrdgsitest -%{_includedir}/xrootd/XrdCl -%{_includedir}/xrootd/XrdPosix -%{_libdir}/libXrdCl.so -%{_libdir}/libXrdFfs.so -%{_libdir}/libXrdPosix.so -%{_mandir}/man1/xrdgsitest.1* - -%files server-libs -%defattr(-,root,root,-) -%{_libdir}/libXrdBwm-5.so -%{_libdir}/libXrdPss-5.so -%{_libdir}/libXrdXrootd-5.so -%{_libdir}/libXrdPfc-5.so -%{_libdir}/libXrdFileCache-5.so -%{_libdir}/libXrdBlacklistDecision-5.so -%{_libdir}/libXrdHttp-5.so -%{_libdir}/libXrdHttpTPC-5.so -%{_libdir}/libXrdHttpUtils.so.2* -%if %{have_macaroons} -%{_libdir}/libXrdMacaroons-5.so -%endif -%{_libdir}/libXrdN2No2p-5.so -%{_libdir}/libXrdOssCsi-5.so -%{_libdir}/libXrdOssSIgpfsT-5.so -%{_libdir}/libXrdServer.so.3* -%{_libdir}/libXrdSsi-5.so -%{_libdir}/libXrdSsiLog-5.so -%{_libdir}/libXrdThrottle-5.so -%{_libdir}/libXrdCmsRedirectLocal-5.so -%{_libdir}/libXrdOfsPrepGPI-5.so - -%files server-devel -%defattr(-,root,root,-) -%{_includedir}/xrootd/XrdAcc -%{_includedir}/xrootd/XrdCms -%{_includedir}/xrootd/XrdPfc -%{_includedir}/xrootd/XrdOss -%{_includedir}/xrootd/XrdOfs -%{_includedir}/xrootd/XrdSfs -%{_includedir}/xrootd/XrdXrootd -%{_includedir}/xrootd/XrdHttp -%{_libdir}/libXrdServer.so -%{_libdir}/libXrdHttpUtils.so - -%files private-devel -%defattr(-,root,root,-) -%{_includedir}/xrootd/private -%{_libdir}/libXrdSsiLib.so -%{_libdir}/libXrdSsiShMap.so -%if %{?_with_isal:1}%{!?_with_isal:0} -%{_libdir}/libXrdEc.so -%endif - -%files client -%defattr(-,root,root,-) -%{_bindir}/xrdadler32 -%{_bindir}/xrdcks -%{_bindir}/xrdcopy -%{_bindir}/xrdcp -%{_bindir}/xrdcrc32c -%{_bindir}/xrdfs -%{_bindir}/xrdgsiproxy -%{_bindir}/xrdmapc -%{_bindir}/xrdpinls -%{_bindir}/xrdreplay -%{_mandir}/man1/xrdadler32.1* -%{_mandir}/man1/xrdcopy.1* -%{_mandir}/man1/xrdcp.1* -%{_mandir}/man1/xrdfs.1* -%{_mandir}/man1/xrdgsiproxy.1* -%{_mandir}/man1/xrdmapc.1* - -%files fuse -%defattr(-,root,root,-) -%{_bindir}/xrootdfs -%{_mandir}/man1/xrootdfs.1* -%dir %{_sysconfdir}/xrootd - -%if %{_with_python2} -%files -n python2-%{name} -%defattr(-,root,root,-) -%{python2_sitearch}/* -%endif - -%if %{_with_python3} -%files -n python%{python3_pkgversion}-%{name} -%defattr(-,root,root,-) -%{python3_sitearch}/* -%endif - -%files voms -%defattr(-,root,root,-) -%{_libdir}/libXrdVoms-5.so -%{_libdir}/libXrdSecgsiVOMS-5.so -%{_libdir}/libXrdHttpVOMS-5.so -%doc %{_mandir}/man1/libXrdVoms.1.gz -%doc %{_mandir}/man1/libXrdSecgsiVOMS.1.gz - -%files doc -%defattr(-,root,root,-) -%doc %{_docdir}/%{name}-%{version} - -%if %{?_with_ceph:1}%{!?_with_ceph:0} -%files ceph -%defattr(-,root,root,-) -%{_libdir}/libXrdCeph-5.so -%{_libdir}/libXrdCephXattr-5.so -%{_libdir}/libXrdCephPosix.so* -%endif - -%if %{?_with_xrdclhttp:1}%{!?_with_xrdclhttp:0} -%files -n xrdcl-http -%defattr(-,root,root,-) -%{_libdir}/libXrdClHttp-5.so -%{_sysconfdir}/xrootd/client.plugins.d/xrdcl-http-plugin.conf -%endif - -%if %{?_with_scitokens:1}%{!?_with_scitokens:0} -%files scitokens -%defattr(-,root,root,-) -%{_libdir}/libXrdAccSciTokens-5.so -%endif - -%if %{?_with_tests:1}%{!?_with_tests:0} -%files tests -%defattr(-,root,root,-) -%{_bindir}/test-runner -%{_bindir}/xrdshmap -%{_libdir}/libXrdClTests.so -%{_libdir}/libXrdClTestsHelper.so -%{_libdir}/libXrdClTestMonitor*.so -%if %{?_with_isal:1}%{!?_with_isal:0} -%{_libdir}/libXrdEcTests.so -%endif -%if %{?_with_ceph:1}%{!?_with_ceph:0} -%{_libdir}/libXrdCephTests*.so -%endif -%endif - -%files selinux -%defattr(-,root,root) -%{_datadir}/selinux/packages/%{name}/%{name}.pp - -%if 0%{?_with_compat} -%files client-compat -# from xrootd-libs: -%{_libdir}/libXrdAppUtils.so.1* -%{_libdir}/libXrdCks*-4.so -%{_libdir}/libXrdClProxyPlugin-4.so -%{_libdir}/libXrdCrypto.so.1* -%{_libdir}/libXrdCryptoLite.so.1* -%{_libdir}/libXrdCryptossl-4.so -%{_libdir}/libXrdSec*-4.so -%{_libdir}/libXrdUtils.so.2* -%{_libdir}/libXrdXml.so.2* - -# from xrootd-client-libs -%{_libdir}/libXrdCl.so.2* -%{_libdir}/libXrdClient.so.2* -%{_libdir}/libXrdFfs.so.2* -%{_libdir}/libXrdPosix.so.2* -%{_libdir}/libXrdPosixPreload.so.1* -%{_libdir}/libXrdSsiLib.so.1* -%{_libdir}/libXrdSsiShMap.so.1* - -%files server-compat -# from server (renamed) -%{_bindir}/cmsd-4 -%{_bindir}/frm_purged-4 -%{_bindir}/frm_xfrd-4 -%{_bindir}/xrootd-4 -# from server-libs -%{_libdir}/libXrdBwm-4.so -%{_libdir}/libXrdPss-4.so -%{_libdir}/libXrdXrootd-4.so -%{_libdir}/libXrdFileCache-4.so -%{_libdir}/libXrdBlacklistDecision-4.so -%{_libdir}/libXrdHttp-4.so -%{_libdir}/libXrdHttpTPC-4.so -%{_libdir}/libXrdHttpUtils.so.1* -%if %{have_macaroons} -%{_libdir}/libXrdMacaroons-4.so -%endif -%{_libdir}/libXrdN2No2p-4.so -%{_libdir}/libXrdOssSIgpfsT-4.so -%{_libdir}/libXrdServer.so.2* -%{_libdir}/libXrdSsi-4.so -%{_libdir}/libXrdSsiLog-4.so -%{_libdir}/libXrdThrottle-4.so -%{_libdir}/libXrdCmsRedirectLocal-4.so -%{_libdir}/libXrdVoms-4.so - -%endif -# end _with_compat - -#------------------------------------------------------------------------------- -# Changelog -#------------------------------------------------------------------------------- -%changelog -* Thu Oct 15 2020 Michal Simon - 5.0.2-1 -- Introduce xrootd-scitokens package - -* Wed May 27 2020 Michal Simon - 4.12.2-1 -- Remove xrootd-voms-devel - -* Fri Apr 17 2020 Michal Simon - 4.12.0-1 -- Introduce xrootd-voms and xrootd-voms-devel packages - -* Mon Sep 02 2019 Michal Simon - 4.10.1-1 -- Move xrdmapc to client package - -* Fri Aug 30 2019 Michal Simon - 5.0.0 -- Remove XRootD 3.x.x compat package - -* Wed Apr 17 2019 Michal Simon - 4.10.0-1 -- Create add xrdcl-http package - -* Tue Jan 08 2019 Edgar Fajardo -- Create config dir /etc/xrootd/config.d - -* Tue May 08 2018 Michal Simon -- Make python3 sub-package optional - -* Fri Nov 10 2017 Michal Simon - 1:4.8.0-1 -- Add python3 sub-package -- Rename python sub-package - -* Tue Dec 13 2016 Gerardo Ganis -- Add xrdgsitest to xrootd-client-devel - -* Mon Mar 16 2015 Lukasz Janyst -- create the python package - -* Wed Mar 11 2015 Lukasz Janyst -- create the xrootd-ceph package - -* Thu Oct 30 2014 Lukasz Janyst -- update for 4.1 and introduce 3.3.6 compat packages - -* Thu Aug 28 2014 Lukasz Janyst -- add support for systemd - -* Wed Aug 27 2014 Lukasz Janyst -- use generic selinux policy build mechanisms - -* Tue Apr 01 2014 Lukasz Janyst -- correct the license field (LGPLv3+) -- rename to xrootd4 -- add 'conflicts' statements -- remove 'provides' and 'obsoletes' - -* Mon Mar 31 2014 Lukasz Janyst -- Add selinux policy - -* Fri Jan 24 2014 Lukasz Janyst -- Import XrdHttp - -* Fri Jun 7 2013 Lukasz Janyst -- adopt the EPEL RPM layout by Mattias Ellert - -* Tue Apr 2 2013 Lukasz Janyst -- remove perl - -* Thu Nov 1 2012 Justin Salmon -- add tests package - -* Fri Oct 21 2011 Lukasz Janyst 3.1.0-1 -- bump the version to 3.1.0 - -* Mon Apr 11 2011 Lukasz Janyst 3.0.3-1 -- the first RPM release - version 3.0.3 -- the detailed release notes are available at: - http://xrootd.org/download/ReleaseNotes.html diff --git a/packaging/wheel/TestCXX14.txt b/packaging/wheel/TestCXX14.txt deleted file mode 100644 index ef1e98e0bbf..00000000000 --- a/packaging/wheel/TestCXX14.txt +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################### -# Test with cmake if the compiler supports all features of C++14 -############################################################################### - -cmake_minimum_required(VERSION 3.16...3.25 FATAL_ERROR) -project(TestCXX14 CXX) - -message("Your C++ compiler supports these C++14 features:") - -foreach( feature ${CMAKE_CXX_COMPILE_FEATURES} ) - - if( feature STREQUAL "cxx_std_14" ) - message( "${feature}" ) - set( has_cxx_std_14 TRUE ) - endif() - - if( feature STREQUAL "cxx_aggregate_default_initializers" ) - message( "${feature}" ) - set( has_cxx_aggregate_default_initializers TRUE ) - endif() - - if( feature STREQUAL "cxx_attribute_deprecated" ) - message( "${feature}" ) - set( has_cxx_attribute_deprecated TRUE ) - endif() - - if( feature STREQUAL "cxx_binary_literals" ) - message( "${feature}" ) - set( has_cxx_binary_literals TRUE ) - endif() - - if( feature STREQUAL "cxx_contextual_conversions" ) - message( "${feature}" ) - set( has_cxx_contextual_conversions TRUE ) - endif() - - if( feature STREQUAL "cxx_decltype_auto" ) - message( "${feature}" ) - set( has_cxx_decltype_auto TRUE ) - endif() - - if( feature STREQUAL "cxx_digit_separators" ) - message( "${feature}" ) - set( has_cxx_digit_separators TRUE ) - endif() - - if( feature STREQUAL "cxx_generic_lambdas" ) - message( "${feature}" ) - set( has_cxx_generic_lambdas TRUE ) - endif() - - if( feature STREQUAL "cxx_lambda_init_captures" ) - message( "${feature}" ) - set( has_cxx_lambda_init_captures TRUE ) - endif() - - if( feature STREQUAL "cxx_relaxed_constexpr" ) - message( "${feature}" ) - set( has_cxx_relaxed_constexpr TRUE ) - endif() - - if( feature STREQUAL "cxx_return_type_deduction" ) - message( "${feature}" ) - set( has_cxx_return_type_deduction TRUE ) - endif() - - if( feature STREQUAL "cxx_variable_templates" ) - message( "${feature}" ) - set( has_cxx_variable_templates TRUE ) - endif() - -endforeach() - -if( has_cxx_std_14 AND - has_cxx_aggregate_default_initializers AND - has_cxx_attribute_deprecated AND - has_cxx_binary_literals AND - has_cxx_contextual_conversions AND - has_cxx_decltype_auto AND - has_cxx_digit_separators AND - has_cxx_generic_lambdas AND - has_cxx_lambda_init_captures AND - has_cxx_relaxed_constexpr AND - has_cxx_return_type_deduction AND - has_cxx_variable_templates ) - - message( "We have full C++14 support." ) - set( has_full_cxx14 TRUE ) - -endif() - -if( NOT has_full_cxx14 ) - message( FATAL_ERROR "The compiler DOES NOT support all features of C++14!." ) -endif() - \ No newline at end of file diff --git a/packaging/wheel/has_c++14.sh b/packaging/wheel/has_c++14.sh deleted file mode 100755 index 25c883fc988..00000000000 --- a/packaging/wheel/has_c++14.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -mkdir has_c++14.tmp -cp packaging/wheel/TestCXX14.txt has_c++14.tmp/CMakeLists.txt -cd has_c++14.tmp -mkdir build -cd build -cmake3 .. -has_cxx14=$? -cd ../.. -rm -rf has_c++14.tmp -exit $has_cxx14 \ No newline at end of file diff --git a/packaging/wheel/install.sh b/packaging/wheel/install.sh deleted file mode 100755 index 897b21cee13..00000000000 --- a/packaging/wheel/install.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -startdir="$(pwd)" -mkdir xrootdbuild -cd xrootdbuild - -# build only client -# build python bindings -# set install prefix -# set the respective version of python -# replace the default BINDIR with a custom one that can be easily removed afterwards -# (for the python bindings we don't want to install the binaries) -CMAKE_ARGS="-DPYPI_BUILD=TRUE -DXRDCL_LIB_ONLY=TRUE -DENABLE_PYTHON=TRUE -DCMAKE_INSTALL_PREFIX=$1/pyxrootd -DXRD_PYTHON_REQ_VERSION=$2 -DCMAKE_INSTALL_BINDIR=$startdir/xrootdbuild/bin" - -if [ "$5" = "true" ]; then - source /opt/rh/devtoolset-7/enable -fi - -cmake_path=$4 -$cmake_path .. $CMAKE_ARGS - -res=$? -if [ "$res" -ne "0" ]; then - exit 1 -fi - -make -j8 -res=$? -if [ "$res" -ne "0" ]; then - exit 1 -fi - -cd src -make install -res=$? -if [ "$res" -ne "0" ]; then - exit 1 -fi - -cd ../bindings/python - -# Determine if shutil.which is available for a modern Python package install -# (shutil.which was added in Python 3.3, so any version of Python 3 now will have it) -# TODO: Drop support for Python 3.3 and simplify to pip approach -${6} -c 'from shutil import which' &> /dev/null # $6 holds the python sys.executable -shutil_which_available=$? -if [ "${shutil_which_available}" -ne "0" ]; then - ${6} setup.py install ${3} - res=$? -else - ${6} -m pip install ${3} . - res=$? -fi -unset shutil_which_available - -if [ "$res" -ne "0" ]; then - exit 1 -fi - -cd $startdir -rm -r xrootdbuild diff --git a/packaging/wheel/publish.sh b/packaging/wheel/publish.sh index ddcb335157d..72c26260bee 100755 --- a/packaging/wheel/publish.sh +++ b/packaging/wheel/publish.sh @@ -1,10 +1,8 @@ #!/bin/bash -./genversion.sh -version=$(./genversion.sh --print-only) -version=${version#v} -echo $version > bindings/python/VERSION -rm -r dist +./genversion.sh >| VERSION + +[[ -d dist ]] && rm -rf dist # Determine if wheel.bdist_wheel is available for wheel.bdist_wheel in setup.py python3 -c 'import wheel' &> /dev/null diff --git a/packaging/wheel/setup.py b/packaging/wheel/setup.py deleted file mode 100644 index 179b57a9aec..00000000000 --- a/packaging/wheel/setup.py +++ /dev/null @@ -1,220 +0,0 @@ -from setuptools import setup -from setuptools.command.install import install -from setuptools.command.sdist import sdist -from wheel.bdist_wheel import bdist_wheel - -# shutil.which was added in Python 3.3 -# c.f. https://docs.python.org/3/library/shutil.html#shutil.which -try: - from shutil import which -except ImportError: - from distutils.spawn import find_executable as which - -import subprocess -import sys - -def get_version(): - version = subprocess.check_output(['./genversion.sh', '--print-only']).decode() - - # Sanitize in keeping with PEP 440 - # c.f. https://www.python.org/dev/peps/pep-0440/ - # c.f. https://github.com/pypa/pip/issues/8368 - # version needs to pass pip._vendor.packaging.version.Version() - version = version.replace("-", ".") - - if version.startswith("v"): - version = version[1:] - - version_parts = version.split(".") - - # Ensure release candidates sanitized to ..rc - if version_parts[-1].startswith("rc"): - version = ".".join(version_parts[:-1]) + version_parts[-1] - version_parts = version.split(".") - - # Assume SemVer as default case - if len(version_parts[0]) == 8: - # CalVer - date = version_parts[0] - year = date[:4] - incremental = date[4:] - if incremental.startswith("0"): - incremental = incremental[1:] - - version = year + "." + incremental - - return version - -def get_version_from_file(): - try: - with open('./bindings/python/VERSION') as f: - version = f.read().split('/n')[0] - return version - except: - print('Failed to get version from file. Using unknown') - return 'unknown' - -def binary_exists(name): - """Check whether `name` is on PATH.""" - return which(name) is not None - -def check_cmake3(path): - args = (path, "--version") - popen = subprocess.Popen(args, stdout=subprocess.PIPE) - popen.wait() - output = popen.stdout.read().decode("utf-8") - prefix_len = len( "cmake version " ) - version = output[prefix_len:].split( '.' ) - return int( version[0] ) >= 3 - -def cmake_exists(): - """Check whether CMAKE is on PATH.""" - path = which('cmake') - if path is not None: - if check_cmake3(path): return True, path - path = which('cmake3') - return path is not None, path - -def is_rhel7(): - """check if we are running on rhel7 platform""" - try: - f = open( '/etc/redhat-release', "r" ) - txt = f.read().split() - i = txt.index( 'release' ) + 1 - return txt[i][0] == '7' - except IOError: - return False - except ValueError: - return False - -def has_devtoolset(): - """check if devtoolset-7 is installed""" - import subprocess - args = ( "/usr/bin/rpm", "-q", "devtoolset-7-gcc-c++" ) - popen = subprocess.Popen(args, stdout=subprocess.PIPE) - rc = popen.wait() - return rc == 0 - -def has_cxx14(): - """check if C++ compiler supports C++14""" - import subprocess - popen = subprocess.Popen("./has_c++14.sh", stdout=subprocess.PIPE) - rc = popen.wait() - return rc == 0 - -# def python_dependency_name( py_version_short, py_version_nodot ): -# """ find the name of python dependency """ -# # this is the path to default python -# path = which( 'python' ) -# from os.path import realpath -# # this is the real default python after resolving symlinks -# real = realpath(path) -# index = real.find( 'python' ) + len( 'python' ) -# # this is the version of default python -# defaultpy = real[index:] -# if defaultpy != py_version_short: -# return 'python' + py_version_nodot -# return 'python' - -class CustomInstall(install): - def run(self): - - py_version_short = self.config_vars['py_version_short'] - py_version_nodot = self.config_vars['py_version_nodot'] - - cmake_bin, cmake_path = cmake_exists() - make_bin = binary_exists( 'make' ) - comp_bin = binary_exists( 'c++' ) or binary_exists( 'g++' ) or binary_exists( 'clang' ) - - import pkgconfig - zlib_dev = pkgconfig.exists( 'zlib' ) - openssl_dev = pkgconfig.exists( 'openssl' ) - uuid_dev = pkgconfig.exists( 'uuid' ) - - if is_rhel7(): - if has_cxx14(): - devtoolset7 = True # we only care about devtoolset7 if the compiler does not support C++14 - need_devtoolset = "false" - else: - devtoolset7 = has_devtoolset() - need_devtoolset = "true" - else: - devtoolset7 = True # we only care about devtoolset7 on rhel7 - need_devtoolset = "false" - - pyname = None - if py_version_nodot[0] == '3': - python_dev = pkgconfig.exists( 'python3' ) or pkgconfig.exists( 'python' + py_version_nodot ); - pyname = 'python3' - else: - python_dev = pkgconfig.exists( 'python' ); - pyname = 'python' - - missing_dep = not ( cmake_bin and make_bin and comp_bin and zlib_dev and openssl_dev and python_dev and uuid_dev and devtoolset7 ) - - if missing_dep: - print( 'Some dependencies are missing:') - if not cmake_bin: print('\tcmake (version 3) is missing!') - if not make_bin: print('\tmake is missing!') - if not comp_bin: print('\tC++ compiler is missing (g++, c++, clang, etc.)!') - if not zlib_dev: print('\tzlib development package is missing!') - if not openssl_dev: print('\topenssl development package is missing!') - if not python_dev: print('\t{} development package is missing!'.format(pyname) ) - if not uuid_dev: print('\tuuid development package is missing') - if not devtoolset7: print('\tdevtoolset-7-gcc-c++ package is missing') - raise Exception( 'Dependencies missing!' ) - - useropt = '' - command = ['./install.sh'] - if self.user: - prefix = self.install_usersite - useropt = '--user' - else: - prefix = self.install_platlib - command.append(prefix) - command.append( py_version_short ) - command.append( useropt ) - command.append( cmake_path ) - command.append( need_devtoolset ) - command.append( sys.executable ) - rc = subprocess.call(command) - if rc: - raise Exception( 'Install step failed!' ) - - -class CustomDist(sdist): - def write_version_to_file(self): - - version = get_version() - with open('bindings/python/VERSION', 'w') as vi: - vi.write(version) - - def run(self): - self.write_version_to_file() - sdist.run(self) - - -class CustomWheelGen(bdist_wheel): - # Do not generate wheel - def run(self): - pass - -version = get_version_from_file() -setup_requires=[ 'pkgconfig' ] - -setup( - name = 'xrootd', - version = version, - author = 'XRootD Developers', - author_email = 'xrootd-dev@slac.stanford.edu', - url = 'http://xrootd.org', - license = 'LGPLv3+', - description = "XRootD with Python bindings", - long_description = "XRootD with Python bindings", - setup_requires = setup_requires, - cmdclass = { - 'install': CustomInstall, - 'sdist': CustomDist, - 'bdist_wheel': CustomWheelGen - } -) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..b0f076532a0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=42"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py new file mode 100644 index 00000000000..c676e0854fb --- /dev/null +++ b/setup.py @@ -0,0 +1,124 @@ +import os +import platform +import subprocess +import sys + +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext +from subprocess import check_call, check_output + +try: + from shutil import which +except ImportError: + from distutils.spawn import find_executable as which + +cmake = which("cmake3") or which("cmake") + +def get_cmake_args(): + args = os.getenv('CMAKE_ARGS') + + if not args: + return [] + + from shlex import split + return split(args) + +def get_version(): + try: + version = open('VERSION').read().strip() + + if version.startswith('$'): + output = check_output(['git', 'describe']) + version = output.decode().strip() + except: + version = None + pass + + if version is None: + from datetime import date + version = '5.7-rc' + date.today().strftime("%Y%m%d") + + if version.startswith('v'): + version = version[1:] + + # Sanitize version to conform to PEP 440 + # https://www.python.org/dev/peps/pep-0440 + version = version.replace('-rc', 'rc') + version = version.replace('-g', '+git.') + version = version.replace('-', '.post', 1) + version = version.replace('-', '.') + + return version + +class CMakeExtension(Extension): + def __init__(self, name, src='.', sources=[], **kwa): + Extension.__init__(self, name, sources=sources, **kwa) + self.src = os.path.abspath(src) + +class CMakeBuild(build_ext): + def build_extensions(self): + if cmake is None: + raise RuntimeError('Cannot find CMake executable') + + for ext in self.extensions: + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + + # Use $ORIGIN RPATH to ensure that the client library can load + # libXrdCl which will be installed in the same directory. Build + # with install RPATH because libraries are installed by Python/pip + # not CMake, so CMake cannot fix the install RPATH later on. + + cmake_args = [ + '-DPython_EXECUTABLE={}'.format(sys.executable), + '-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE', + '-DCMAKE_ARCHIVE_OUTPUT_DIRECTORY={}/{}'.format(self.build_temp, ext.name), + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}/{}'.format(extdir, ext.name), + '-DENABLE_PYTHON=1', '-DENABLE_XRDCL=1', '-DXRDCL_LIB_ONLY=1', '-DPYPI_BUILD=1' + ] + + if sys.platform == 'darwin': + cmake_args += [ '-DCMAKE_INSTALL_RPATH=@loader_path' ] + else: + cmake_args += [ '-DCMAKE_INSTALL_RPATH=$ORIGIN' ] + + cmake_args += get_cmake_args() + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + check_call([cmake, ext.src, '-B', self.build_temp] + cmake_args) + check_call([cmake, '--build', self.build_temp, '--parallel']) + +version = get_version() + +setup(name='xrootd', + version=version, + description='eXtended ROOT daemon', + author='XRootD Developers', + author_email='xrootd-dev@slac.stanford.edu', + url='http://xrootd.org', + download_url='https://github.com/xrootd/xrootd/archive/v%s.tar.gz' % version, + keywords=['XRootD', 'network filesystem'], + license='LGPLv3+', + long_description=open('README.md').read(), + long_description_content_type='text/markdown', + packages = ['XRootD', 'XRootD.client', 'pyxrootd'], + package_dir = { + 'pyxrootd' : 'bindings/python/src', + 'XRootD' : 'bindings/python/libs', + 'XRootD/client': 'bindings/python/libs/client', + }, + ext_modules= [ CMakeExtension('pyxrootd') ], + cmdclass={ 'build_ext': CMakeBuild }, + zip_safe=False, + classifiers=[ + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + "Operating System :: Unix", + "Programming Language :: C++", + "Programming Language :: Python", + ] + ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fd1cde70dd6..69add8bd96f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -55,10 +55,7 @@ if( NOT XRDCL_ONLY ) include( XrdPlugins ) include( XrdSsi ) include( XrdPfc ) - - if( CMAKE_COMPILER_IS_GNUCXX ) - include( XrdOssCsi ) - endif() + include( XrdOssCsi ) if( BUILD_HTTP ) include( XrdHttp ) @@ -82,56 +79,3 @@ if( NOT XRDCL_ONLY ) endif() endif() - -#------------------------------------------------------------------------------- -# Install the utility scripts -#------------------------------------------------------------------------------- -if( NOT XRDCL_ONLY ) -install( - FILES - ${CMAKE_SOURCE_DIR}/utils/XrdCmsNotify.pm - ${CMAKE_SOURCE_DIR}/utils/netchk - ${CMAKE_SOURCE_DIR}/utils/XrdOlbMonPerf - ${CMAKE_SOURCE_DIR}/utils/cms_monPerf - DESTINATION ${CMAKE_INSTALL_DATADIR}/xrootd/utils - PERMISSIONS - OWNER_EXECUTE OWNER_WRITE OWNER_READ - GROUP_EXECUTE GROUP_READ - WORLD_EXECUTE WORLD_READ ) -endif() - -if( NOT XRDCL_LIB_ONLY ) -#------------------------------------------------------------------------------- -# Install xrootd-config -#------------------------------------------------------------------------------- -install( - CODE " - EXECUTE_PROCESS( - COMMAND cat ${CMAKE_SOURCE_DIR}/utils/xrootd-config - COMMAND sed -e \"s/__VERSION__/${XROOTD_VERSION}/\" - COMMAND sed -e \"s|__INCLUDEDIR__|${CMAKE_INSTALL_INCLUDEDIR}|\" - COMMAND sed -e \"s/__PLUGIN_VERSION__/${PLUGIN_VERSION}/\" - COMMAND sed -e \"s|__PREFIX__|${CMAKE_INSTALL_PREFIX}|\" - OUTPUT_FILE \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/xrootd-config ) - EXECUTE_PROCESS( - COMMAND chmod 755 \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/xrootd-config )" -) - -#------------------------------------------------------------------------------- -# Post process man pages -#------------------------------------------------------------------------------- -install( - CODE " - FILE(GLOB MANPAGES - \"\$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man[1,8]/*.[1,8]\" ) - FOREACH(MANPAGE \${MANPAGES}) - MESSAGE( \"-- Processing: \" \${MANPAGE} ) - EXECUTE_PROCESS( - COMMAND cat \${MANPAGE} - COMMAND sed -e \"s/__VERSION__/${XROOTD_VERSION}/\" - OUTPUT_FILE \${MANPAGE}.new ) - EXECUTE_PROCESS( - COMMAND mv -f \${MANPAGE}.new \${MANPAGE} ) - ENDFOREACH()" -) -endif() diff --git a/src/XProtocol/XProtocol.cc b/src/XProtocol/XProtocol.cc index dd15990c060..1e1772d3fb0 100644 --- a/src/XProtocol/XProtocol.cc +++ b/src/XProtocol/XProtocol.cc @@ -203,7 +203,7 @@ char* ClientFattrRequest::VVecInsert( const char *value, char *buffer ) // char* ClientFattrRequest::NVecRead( char* buffer, kXR_unt16 &rc ) { - rc = *reinterpret_cast( buffer ); + memcpy(&rc, buffer, sizeof(kXR_unt16)); rc = htons( rc ); buffer += sizeof( kXR_unt16 ); return buffer; @@ -222,7 +222,7 @@ char* ClientFattrRequest::NVecRead( char* buffer, char *&name ) // char* ClientFattrRequest::VVecRead( char* buffer, kXR_int32 &len ) { - len = *reinterpret_cast( buffer ); + memcpy(&len, buffer, sizeof(kXR_int32)); len = htonl( len ); buffer += sizeof( kXR_int32 ); return buffer; diff --git a/src/XProtocol/XProtocol.hh b/src/XProtocol/XProtocol.hh index eb9af2c7953..561161d9913 100644 --- a/src/XProtocol/XProtocol.hh +++ b/src/XProtocol/XProtocol.hh @@ -1370,7 +1370,12 @@ static int mapError(int rc) case ENOTBLK: return kXR_NotFile; case ENOTSUP: return kXR_Unsupported; case EISDIR: return kXR_isDirectory; - case EEXIST: return kXR_ItExists; + case ENOTEMPTY: [[fallthrough]]; + // In the case one tries to delete a non-empty directory + // we have decided that until the next major release + // the kXR_ItExists flag will be returned + case EEXIST: + return kXR_ItExists; case EBADRQC: return kXR_InvalidRequest; case ETXTBSY: return kXR_inProgress; case ENODEV: return kXR_FSError; diff --git a/src/Xrd/XrdConfig.cc b/src/Xrd/XrdConfig.cc index f838f214046..73c91d45b41 100644 --- a/src/Xrd/XrdConfig.cc +++ b/src/Xrd/XrdConfig.cc @@ -93,6 +93,9 @@ #if defined(__linux__) || defined(__GNU__) #include #endif +#if defined(__linux__) +#include +#endif #ifdef __APPLE__ #include #endif @@ -274,6 +277,8 @@ XrdConfig::XrdConfig() NetADM = 0; coreV = 1; Specs = 0; + isStrict = false; + maxFD = 256*1024; // 256K default Firstcp = Lastcp = 0; @@ -462,7 +467,7 @@ int XrdConfig::Configure(int argc, char **argv) Log.Emsg("Config", buff, "parameter not specified."); Usage(1); break; - case 'v': cerr < maxFD) rlim.rlim_cur = maxFD; + if (rlim.rlim_max == RLIM_INFINITY || (isStrict && rlim.rlim_max > maxFD)) + rlim.rlim_cur = maxFD; else rlim.rlim_cur = rlim.rlim_max; #if (defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_5)) if (rlim.rlim_cur > OPEN_MAX) rlim.rlim_max = rlim.rlim_cur = OPEN_MAX; +#endif +#if defined(__linux__) +// Setting a limit beyond this value on Linux is guaranteed to fail during epoll_wait() + unsigned int epoll_max_fd = (INT_MAX / sizeof(struct epoll_event)); + if (rlim.rlim_cur > (rlim_t)epoll_max_fd) rlim.rlim_max = rlim.rlim_cur = epoll_max_fd; #endif if (setrlimit(RLIMIT_NOFILE, &rlim) < 0) return Log.Emsg("Config", errno,"set FD limit"); @@ -1468,11 +1480,11 @@ void XrdConfig::Usage(int rc) { extern const char *XrdLicense; - if (rc < 0) cerr <] [-d] [-h] [-H] [-I {v4|v6}]\n" + std::cerr <<"\nUsage: " <] [-d] [-h] [-H] [-I {v4|v6}]\n" "[-k {n|sz|sig}] [-l [=]] [-n name] [-p ] [-P ] [-L ]\n" - "[-R] [-s pidfile] [-S site] [-v] [-z] []" <]" < 0 ? rc : 0); } @@ -1649,6 +1661,46 @@ int XrdConfig::xbuf(XrdSysError *eDest, XrdOucStream &Config) return 0; } + +/******************************************************************************/ +/* x m a x f d */ +/******************************************************************************/ + +/* Function: xmaxfd + + Purpose: To parse the directive: maxfd [strict] + + strict when specified, the limits is always applied. Otherwise, + it is only applied when rlimit is infinite. + maximum number of fs that can be established. + Specify a value optionally suffixed with 'k'. + + Output: 0 upon success or !0 upon failure. +*/ +int XrdConfig::xmaxfd(XrdSysError *eDest, XrdOucStream &Config) +{ + long long minV = 1024, maxV = 1024LL*1024LL; // between 1k and 1m + long long fdVal; + char *val; + + if ((val = Config.GetWord())) + {if (!strcmp(val, "strict")) + {isStrict = true; + val = Config.GetWord(); + } else isStrict = false; + } + + if (!val) + {eDest->Emsg("Config", "file descriptor limit not specified"); return 1;} + + + if (XrdOuca2x::a2sz(*eDest,"maxfd value",val,&fdVal,minV,maxV)) return 1; + + maxFD = static_cast(fdVal); + + return 0; +} + /******************************************************************************/ /* x n e t */ /******************************************************************************/ diff --git a/src/Xrd/XrdConfig.hh b/src/Xrd/XrdConfig.hh index 2d1403a5e10..f29dc6005b2 100644 --- a/src/Xrd/XrdConfig.hh +++ b/src/Xrd/XrdConfig.hh @@ -76,6 +76,7 @@ int xallow(XrdSysError *edest, XrdOucStream &Config); int xapath(XrdSysError *edest, XrdOucStream &Config); int xhpath(XrdSysError *edest, XrdOucStream &Config); int xbuf(XrdSysError *edest, XrdOucStream &Config); +int xmaxfd(XrdSysError *edest, XrdOucStream &Config); int xnet(XrdSysError *edest, XrdOucStream &Config); int xnkap(XrdSysError *edest, char *val); int xlog(XrdSysError *edest, XrdOucStream &Config); @@ -134,5 +135,8 @@ char ppNet; signed char coreV; char Specs; static const int hpSpec = 0x01; + +bool isStrict; +unsigned int maxFD; }; #endif diff --git a/src/Xrd/XrdLink.hh b/src/Xrd/XrdLink.hh index 3d11e9d6832..5f6a997bdfe 100644 --- a/src/Xrd/XrdLink.hh +++ b/src/Xrd/XrdLink.hh @@ -162,7 +162,7 @@ static XrdLink *Find(int &curr, XrdLinkMatch *who=0); //! criterea (typically, client name and host name). If the //! pointer is nil, a match always occurs. //! -//! @return !0 The length of teh name placed in the buffer. +//! @return !0 The length of the name placed in the buffer. //! =0 No more links exist with the specified criterea. //----------------------------------------------------------------------------- diff --git a/src/Xrd/XrdLinkCtl.hh b/src/Xrd/XrdLinkCtl.hh index 1fee8ac5d92..66df9abb4ab 100644 --- a/src/Xrd/XrdLinkCtl.hh +++ b/src/Xrd/XrdLinkCtl.hh @@ -113,7 +113,7 @@ static XrdPollInfo *fd2PollInfo(int fd) //! @param curr Is an internal tracking value that allows repeated calls. //! It must be set to a value of 0 or less on the initial call //! and not touched therafter unless a null pointer is returned. -//! @param who If the object use to check if teh link matches the wanted +//! @param who If the object use to check if the link matches the wanted //! criterea (typically, client name and host name). If the //! ppointer is nil, the next link is always returned. //! diff --git a/src/Xrd/XrdLinkMatch.hh b/src/Xrd/XrdLinkMatch.hh index 1bd36e5add9..76ce4b22d24 100644 --- a/src/Xrd/XrdLinkMatch.hh +++ b/src/Xrd/XrdLinkMatch.hh @@ -29,8 +29,7 @@ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ -#include -#include +#include class XrdLinkMatch { diff --git a/src/Xrd/XrdMain.cc b/src/Xrd/XrdMain.cc index 29446a5ca12..04177d45c48 100644 --- a/src/Xrd/XrdMain.cc +++ b/src/Xrd/XrdMain.cc @@ -165,6 +165,16 @@ int main(int argc, char *argv[]) char buff[128]; int i, retc; +// Set TZ environment variable to read the system timezone from /etc/localtime +// if it is not already set, to avoid race conditions between localtime_r() and +// mktime() during the multi-threaded phase of the initialization. + + if (access("/etc/localtime", R_OK) == 0) + setenv("TZ", ":/etc/localtime", /* overwrite */ false); + +// Call tzset() early to ensure thread-safety of localtime_r() and mktime(). + tzset(); + // Turn off sigpipe and host a variety of others before we start any threads // XrdSysUtils::SigBlock(); diff --git a/src/Xrd/XrdScheduler.cc b/src/Xrd/XrdScheduler.cc index 75925a9456d..087613e07c2 100644 --- a/src/Xrd/XrdScheduler.cc +++ b/src/Xrd/XrdScheduler.cc @@ -122,7 +122,7 @@ XrdScheduler::XrdScheduler(XrdSysError *eP, XrdOucTrace *tP, // XrdScheduler::XrdScheduler(int minw, int maxw, int maxi) : XrdJob("underused thread monitor"), - WorkAvail(0, "sched work") + XrdTraceOld(0), WorkAvail(0, "sched work") { XrdSysLogger *Logger; int eFD; diff --git a/src/Xrd/XrdTcpMonPin.hh b/src/Xrd/XrdTcpMonPin.hh index bdaa83b1cb7..6d4e9b7a2e4 100644 --- a/src/Xrd/XrdTcpMonPin.hh +++ b/src/Xrd/XrdTcpMonPin.hh @@ -42,6 +42,8 @@ @endcode */ +class XrdNetAddrInfo; + class XrdTcpMonPin { public: diff --git a/src/XrdAcc/XrdAccAuthFile.cc b/src/XrdAcc/XrdAccAuthFile.cc index 1f9601af480..8b738baa733 100644 --- a/src/XrdAcc/XrdAccAuthFile.cc +++ b/src/XrdAcc/XrdAccAuthFile.cc @@ -157,9 +157,11 @@ char XrdAccAuthFile::getID(char **id) return 0; } -// Id's are of the form 'c:', make sure we have that (don't validate it) +// Id's are of the form 'c', but historically they were 'c:' so we accept a +// two character specification but only validate the first to be backward +// compatible. // - if (strlen(pp) != 2 || !index("ghoru", *pp)) + if (strlen(pp) > 2 || !index("ghoru", *pp)) {Eroute->Emsg("AuthFile", "Invalid ID sprecifier -", pp); flags = (DBflags)(flags | dbError); return 0; diff --git a/src/XrdAcc/XrdAccAuthorize.hh b/src/XrdAcc/XrdAccAuthorize.hh index 54f34076332..ff69f0359d2 100644 --- a/src/XrdAcc/XrdAccAuthorize.hh +++ b/src/XrdAcc/XrdAccAuthorize.hh @@ -150,12 +150,15 @@ virtual ~XrdAccAuthorize() {} //! XrdAccAuthorizeObject() is an extern "C" function that is called to obtain //! an instance of the auth object that will be used for all subsequent //! authorization decisions. It must be defined in the plug-in shared library. +//! A second version which is used preferentially if it exists should be +//! used if accessto theenvironmental pointer s needed. //! All the following extern symbols must be defined at file level! //! //! @param lp -> XrdSysLogger to be tied to an XrdSysError object for messages //! @param cfn -> The name of the configuration file //! @param parm -> Parameters specified on the authlib directive. If none it //! is zero. +//! @param envP -> Pointer to environment only available for version 2. //! //! @return Success: A pointer to the authorization object. //! Failure: Null pointer which causes initialization to fail. @@ -170,6 +173,20 @@ typedef XrdAccAuthorize *(*XrdAccAuthorizeObject_t)(XrdSysLogger *lp, const char *cfn, const char *parm) {...} */ + +// Alternatively: + +typedef XrdAccAuthorize *(*XrdAccAuthorizeObject2_t)(XrdSysLogger *lp, + const char *cfn, + const char *parm, + XrdOucEnv *envP); + + +/*! extern "C" XrdAccAuthorize *XrdAccAuthorizeObject2(XrdSysLogger *lp, + const char *cfn, + const char *parm, + XrdOucEnv *envP) {...} +*/ //------------------------------------------------------------------------------ //! Add an authorization object as a wrapper to the existing object. diff --git a/src/XrdAcc/XrdAccGroups.cc b/src/XrdAcc/XrdAccGroups.cc index 94b8edbc75e..c16c2644fa6 100644 --- a/src/XrdAcc/XrdAccGroups.cc +++ b/src/XrdAcc/XrdAccGroups.cc @@ -117,7 +117,7 @@ char *XrdAccGroups::AddName(const XrdAccGroupType gtype, const char *name) if (!(np = hp->Find(name))) {hp->Add(name, 0, 0, Hash_data_is_key); if (!(np = hp->Find(name))) - cerr <<"XrdAccGroups: Unable to add group " <= NGROUPS_MAX) {if (gtabi == NGROUPS_MAX) - cerr <<"XrdAccGroups: More than " <gtabi >= NGROUPS_MAX) {if (grp->gtabi == NGROUPS_MAX) - cerr <<"XrdAccGroups: More than " <gtabi <<"netgroups for " <user <gtabi <<"netgroups for " <user <] [ | ] \n\n"; - cerr <<": -a -g -h -o -r -u \n"; - cerr <<": [ [...]]\n"; - cerr <<": cr - create mv - rename st - status lk - lock\n"; - cerr <<" rd - read wr - write ls - readdir rm - remove\n"; - cerr <<" ec - excl create ei - excl rename\n"; - cerr <<" * - zap args ? - display privs\n"; - cerr <] [ | ] \n\n"; + std::cerr <<": -a -g -h -o -r -u \n"; + std::cerr <<": [ [...]]\n"; + std::cerr <<": cr - create mv - rename st - status lk - lock\n"; + std::cerr <<" rd - read wr - write ls - readdir rm - remove\n"; + std::cerr <<" ec - excl create ei - excl rename\n"; + std::cerr <<" * - zap args ? - display privs\n"; + std::cerr << std::flush; exit(msg ? 1 : 0); } @@ -212,7 +212,7 @@ bool singleshot=false; // Obtain the authorization object // if (!(Authorize = XrdAccDefaultAuthorizeObject(&myLogger, ConfigFN, 0, myVer))) - {cerr << "testaccess: Initialization failed." < [-h ] [-n ] [-x ] []" - "\n: [[pfx]*] | [*[sfx]] []" < [-h ] [-n ] [-x ] []" + "\n: [[pfx]*] | [*[sfx]] []" < + using namespace XrdCl; namespace xrdcl_proxy @@ -45,7 +47,7 @@ public: //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- - virtual ~ProxyPrefixFile(); + virtual ~ProxyPrefixFile() override; //---------------------------------------------------------------------------- //! Open @@ -54,13 +56,13 @@ public: OpenFlags::Flags flags, Access::Mode mode, ResponseHandler* handler, - uint16_t timeout); + uint16_t timeout) override; //---------------------------------------------------------------------------- //! Close //---------------------------------------------------------------------------- virtual XRootDStatus Close(ResponseHandler* handler, - uint16_t timeout) + uint16_t timeout) override { return pFile->Close(handler, timeout); } @@ -70,7 +72,7 @@ public: //---------------------------------------------------------------------------- virtual XRootDStatus Stat(bool force, ResponseHandler* handler, - uint16_t timeout) + uint16_t timeout) override { return pFile->Stat(force, handler, timeout); } @@ -83,11 +85,23 @@ public: uint32_t size, void* buffer, ResponseHandler* handler, - uint16_t timeout) + uint16_t timeout) override { return pFile->Read(offset, size, buffer, handler, timeout); } + //------------------------------------------------------------------------ + //! PgRead + //------------------------------------------------------------------------ + virtual XRootDStatus PgRead( uint64_t offset, + uint32_t size, + void *buffer, + ResponseHandler *handler, + uint16_t timeout ) override + { + return pFile->PgRead(offset, size, buffer, handler, timeout); + } + //---------------------------------------------------------------------------- //! Write //---------------------------------------------------------------------------- @@ -95,16 +109,53 @@ public: uint32_t size, const void* buffer, ResponseHandler* handler, - uint16_t timeout) + uint16_t timeout) override { return pFile->Write(offset, size, buffer, handler, timeout); } + //------------------------------------------------------------------------ + //! Write + //------------------------------------------------------------------------ + virtual XRootDStatus Write( uint64_t offset, + Buffer &&buffer, + ResponseHandler *handler, + uint16_t timeout = 0 ) override + { + return pFile->Write(offset, std::move(buffer), handler, timeout); + } + + //------------------------------------------------------------------------ + //! Write + //------------------------------------------------------------------------ + virtual XRootDStatus Write( uint64_t offset, + uint32_t size, + Optional fdoff, + int fd, + ResponseHandler *handler, + uint16_t timeout = 0 ) override + { + return pFile->Write(offset, size, fdoff, fd, handler, timeout); + } + + //------------------------------------------------------------------------ + //! PgWrite + //------------------------------------------------------------------------ + virtual XRootDStatus PgWrite( uint64_t offset, + uint32_t nbpgs, + const void *buffer, + std::vector &cksums, + ResponseHandler *handler, + uint16_t timeout ) override + { + return pFile->PgWrite(offset, nbpgs, buffer, cksums, handler, timeout); + } + //---------------------------------------------------------------------------- //! Sync //---------------------------------------------------------------------------- virtual XRootDStatus Sync(ResponseHandler* handler, - uint16_t timeout) + uint16_t timeout) override { return pFile->Sync(handler, timeout); } @@ -114,7 +165,7 @@ public: //---------------------------------------------------------------------------- virtual XRootDStatus Truncate(uint64_t size, ResponseHandler* handler, - uint16_t timeout) + uint16_t timeout) override { return pFile->Truncate(size, handler, timeout); } @@ -125,17 +176,39 @@ public: virtual XRootDStatus VectorRead(const ChunkList& chunks, void* buffer, ResponseHandler* handler, - uint16_t timeout) + uint16_t timeout) override { return pFile->VectorRead(chunks, buffer, handler, timeout); } + //------------------------------------------------------------------------ + //! VectorWrite + //------------------------------------------------------------------------ + virtual XRootDStatus VectorWrite( const ChunkList &chunks, + ResponseHandler *handler, + uint16_t timeout = 0 ) override + { + return pFile->VectorWrite(chunks, handler, timeout); + } + + //------------------------------------------------------------------------ + //! @see XrdCl::File::WriteV + //------------------------------------------------------------------------ + virtual XRootDStatus WriteV( uint64_t offset, + const struct iovec *iov, + int iovcnt, + ResponseHandler *handler, + uint16_t timeout = 0 ) override + { + return pFile->WriteV(offset, iov, iovcnt, handler, timeout); + } + //---------------------------------------------------------------------------- //! Fcntl //---------------------------------------------------------------------------- virtual XRootDStatus Fcntl(const Buffer& arg, ResponseHandler* handler, - uint16_t timeout) + uint16_t timeout) override { return pFile->Fcntl(arg, handler, timeout); } @@ -144,7 +217,7 @@ public: //! Visa //---------------------------------------------------------------------------- virtual XRootDStatus Visa(ResponseHandler* handler, - uint16_t timeout) + uint16_t timeout) override { return pFile->Visa(handler, timeout); } @@ -152,7 +225,7 @@ public: //---------------------------------------------------------------------------- //! IsOpen //---------------------------------------------------------------------------- - virtual bool IsOpen() const + virtual bool IsOpen() const override { return pFile->IsOpen(); } @@ -161,7 +234,7 @@ public: //! SetProperty //---------------------------------------------------------------------------- virtual bool SetProperty(const std::string& name, - const std::string& value) + const std::string& value) override { return pFile->SetProperty(name, value); } @@ -170,7 +243,7 @@ public: //! GetProperty //---------------------------------------------------------------------------- virtual bool GetProperty(const std::string& name, - std::string& value) const + std::string& value) const override { return pFile->GetProperty(name, value); } diff --git a/src/XrdApps/XrdClRecordPlugin/README.md b/src/XrdApps/XrdClRecordPlugin/README.md index 8ba35867fa0..5e878a5c0a3 100644 --- a/src/XrdApps/XrdClRecordPlugin/README.md +++ b/src/XrdApps/XrdClRecordPlugin/README.md @@ -32,7 +32,7 @@ _________________ The **xrdreplay** application provides the following operation modes: * print mode (-p) : display runtime and IO statistics for a record file -* verify mode (-v) : verify the existance of the required input files for a record file +* verify mode (-v) : verify the existence of the required input files for a record file * creation mode (-c,-t) : create the required input data using file creation and write the minimal required size (-c) or truncate files to the minimal required size (-t) * playback mode (default) : replay a given record file @@ -113,7 +113,7 @@ xrdreplay -v recording.cvs # size: 536.87 MB [ 0 B out of 536.87 MB ] ( 0.00% ) # ---> info: file exists and has sufficient size ``` -On success the shell returns 0, if there was a missing, too small or inaccesible file it returns -5 (251). +On success the shell returns 0, if there was a missing, too small or inaccessible file it returns -5 (251). ```bash Warning: xrdreplay considers a file only as an input file if it has no bytes written. @@ -250,12 +250,12 @@ usage: xrdreplay [-p|--print] [-c|--create-data] [t|--truncate-data] [-l|--long] -p | --print : print only mode - shows all the IO for the given replay file without actually running any IO -s | --summary : print summary - shows all the aggregated IO counter summed for all files -l | --long : print long - show all file IO counter for each individual file - -v | --verify : verify the existance of all input files + -v | --verify : verify the existence of all input files -x | --speed : change playback speed by factor [ > 0.0 ] -r | --replace := : replace in the argument list the string with - option is usable several times e.g. to change storage prefixes or filenames - [recordfilename] : if a file is given, it will be used as record input otherwhise STDIN is used to read records! + [recordfilename] : if a file is given, it will be used as record input otherwise STDIN is used to read records! example: ... --replace file:://localhost:=root://xrootd.eu/ : redirect local file to remote ``` diff --git a/src/XrdApps/XrdClRecordPlugin/XrdClActionMetrics.hh b/src/XrdApps/XrdClRecordPlugin/XrdClActionMetrics.hh index e5802b771db..6d7351257c8 100644 --- a/src/XrdApps/XrdClRecordPlugin/XrdClActionMetrics.hh +++ b/src/XrdApps/XrdClRecordPlugin/XrdClActionMetrics.hh @@ -65,7 +65,7 @@ struct ActionMetrics std::string cnt = i + "::n"; // IOPS std::string vol = i + "::b"; // number of bytes - std::string err = i + "::e"; // number of unsuccessfull IOs + std::string err = i + "::e"; // number of unsuccessful IOs std::string off = i + "::o"; // maximum offset seen ios[cnt] = 0; diff --git a/src/XrdApps/XrdClRecordPlugin/XrdClReplay.cc b/src/XrdApps/XrdClRecordPlugin/XrdClReplay.cc index b5abbd7b386..4e884dcb47d 100644 --- a/src/XrdApps/XrdClRecordPlugin/XrdClReplay.cc +++ b/src/XrdApps/XrdClRecordPlugin/XrdClReplay.cc @@ -247,7 +247,7 @@ bool AssureFile(const std::string& url, uint64_t size, bool viatruncate, bool ve if (verify) { - std::cerr << "Verify: file is missing or inaccesible: " << url << std::endl; + std::cerr << "Verify: file is missing or inaccessible: " << url << std::endl; return false; } @@ -1079,7 +1079,7 @@ void usage() << " -f | --suppress : force to run all IO with all successful result status - suppress all others" << std::endl; std::cerr - << " - by default the player won't run with an unsuccessfull recorded IO" + << " - by default the player won't run with an unsuccessfully recorded IO" << std::endl; std::cerr << std::endl; std::cerr @@ -1145,10 +1145,10 @@ int main(int argc, char** argv) if (sampling_error) { - std::cerr << "Warning: IO file contains unsuccessfull samples!" << std::endl; + std::cerr << "Warning: IO file contains unsuccessful samples!" << std::endl; if (!opt.suppress_error()) { - std::cerr << "... run with [-f] or [--suppress] option to suppress unsuccessfull IO events!" + std::cerr << "... run with [-f] or [--suppress] option to suppress unsuccessful IO events!" << std::endl; exit(-1); } diff --git a/src/XrdApps/XrdClRecordPlugin/XrdClReplayArgs.hh b/src/XrdApps/XrdClRecordPlugin/XrdClReplayArgs.hh index 04edb34fbe6..898386b93ee 100644 --- a/src/XrdApps/XrdClRecordPlugin/XrdClReplayArgs.hh +++ b/src/XrdApps/XrdClRecordPlugin/XrdClReplayArgs.hh @@ -163,7 +163,7 @@ class ReplayArgs std::cerr << " -l | --long : print long - show all file IO counter for each individual file" << std::endl; - std::cerr << " -v | --verify : verify the existance of all input files" + std::cerr << " -v | --verify : verify the existence of all input files" << std::endl; std::cerr << " -x | --speed : change playback speed by factor [ > 0.0 ]" @@ -176,7 +176,7 @@ class ReplayArgs << std::endl; std::cerr << std::endl; std::cerr - << " [recordfilename] : if a file is given, it will be used as record input otherwhise STDIN is used to read records!" + << " [recordfilename] : if a file is given, it will be used as record input otherwise STDIN is used to read records!" << std::endl; std::cerr << "example: ... --replace file:://localhost:=root://xrootd.eu/ : redirect local file to remote" diff --git a/src/XrdApps/XrdCpConfig.cc b/src/XrdApps/XrdCpConfig.cc index 3008f4c435d..5981c4a9242 100644 --- a/src/XrdApps/XrdCpConfig.cc +++ b/src/XrdApps/XrdCpConfig.cc @@ -53,7 +53,7 @@ using namespace std; /* D e f i n e M a c r o s */ /******************************************************************************/ -#define EMSG(x) cerr < ] [-DS ] [-np]\n" " [-md5] [-OD] [-OS] [-version] [-x]"; - cerr <<(Opts & opt1Src ? Syntax1 : Syntax) < | -}\n" + std::cerr <<"\nUsage: xrdcrc32c [opts] [ | -]\n" "\n the path to the file whose checksum if to be computed." "\n- compute checksum from data presented at standard in;" "\n example: xrdcp - | xrdcrc32c -\n" @@ -81,7 +81,7 @@ void Usage(int rc) "\n-n do not end output with a newline character." "\n-s do not include file path in output result." "\n-x do not print leading zeroes in the checksum, if any." - <= argc) - {cerr <hasfile; pfxbuff[2] = clnow->verfile; } - cout <<' ' <name <state <name <state <nextSrv; } } @@ -410,7 +410,7 @@ void PrintMap(clMap *clmP, int lvl) if (lvl) pfxbuff[2] = ' '; while(clnow) {if (lvl) pfxbuff[1] = clnow->hasfile; - cout <name <state <name <state <valid && clnow->nextLvl) PrintMap(clnow->nextLvl,lvl+1); clnow = clnow->nextMan; } @@ -451,18 +451,18 @@ namespace void Usage(const char *emsg) { if (emsg) EMSG(emsg); - cerr <<"Usage: xrdmapc [] : []\n" - <<": [--help] [--list {all|m|s}] [--quiet] [--refresh] [--verify]" <] : []\n" + <<": [--help] [--list {all|m|s}] [--quiet] [--refresh] [--verify]" < existence status at each server.\n" " when specified, uses : to determine the locations\n" " of path and does optional verification." - <name <state <name <state <name - <<" referred to the following unconnected node:" <name + <<" referred to the following unconnected node:" <name <name <nextSrv; } } diff --git a/src/XrdApps/XrdMpxStats.cc b/src/XrdApps/XrdMpxStats.cc index 8268a652853..e9e53498390 100644 --- a/src/XrdApps/XrdMpxStats.cc +++ b/src/XrdApps/XrdMpxStats.cc @@ -198,7 +198,7 @@ void *mainOutput(void *parg) void Usage(int rc) { - cerr <<"\nUsage: mpxstats [-f {cgi|flat|xml}] -p [-s]" < [-s]" <[:]\n" + std::cerr <<"\nUsage: xrdqstats [opts] [:]\n" "\nopts: -f {cgi|flat|xml} -h -i -n -s what -z\n" "\n-f specify display format (default is wordy text format)." "\n-i number of seconds to wait before between redisplays, default 10." @@ -77,7 +77,7 @@ void Usage(int rc) "\na - All (default) b - Buffer usage d - Device polling" "\ni - Identification c - Connections p - Protocols" "\ns - Scheduling u - Usage data z - Synchronized info" - <= argc) - {cerr <GetBuffer() <GetBuffer() <Format(0, theStats->GetBuffer(), obuff); char *bP = obuff; while(wLen > 0) @@ -198,7 +198,7 @@ int main(int argc, char *argv[]) } delete theStats; if (WTime) sleep(WTime); - if (Count) cout <<"\n"; + if (Count) std::cout <<"\n"; } // All done diff --git a/src/XrdApps/XrdWait41.cc b/src/XrdApps/XrdWait41.cc index 3b5b48d164e..80cce27484f 100644 --- a/src/XrdApps/XrdWait41.cc +++ b/src/XrdApps/XrdWait41.cc @@ -140,7 +140,7 @@ int main(int argc, char *argv[]) for (i = 1; i < argc; i++) {if (stat(argv[i], &Stat)) {eText = XrdSysE2T(errno); - cerr <<"wait41: " <d_name); if (stat(buff, &Stat)) {eText = XrdSysE2T(errno); - cerr <<"wait41: " <text <text <val = open(gfP->text, O_CREAT|O_RDWR, AMode)) < 0) {eTxt = XrdSysE2T(errno); - cerr <<"Wait41: " <text <text <text <text <val); } else Num++; gfP = gfP->next; diff --git a/src/XrdBwm/XrdBwmTrace.hh b/src/XrdBwm/XrdBwmTrace.hh index d321e814d5e..d994089c0cf 100644 --- a/src/XrdBwm/XrdBwmTrace.hh +++ b/src/XrdBwm/XrdBwmTrace.hh @@ -40,7 +40,7 @@ extern XrdOucTrace BwmTrace; #define GTRACE(act) BwmTrace.What & TRACE_ ## act #define TRACES(x) \ - {BwmTrace.Beg(epname,tident); cerr <> /etc/yum.repos.d/ceph.repo + - echo -e '[xrootd-testing]\nname=XRootD Testing repository\nbaseurl=http://xrootd.org/binaries/testing/slc/7/$basearch http://xrootd.cern.ch/sw/repos/testing/slc/7/$basearch\ngpgcheck=1\nenabled=1\nprotect=0\ngpgkey=http://xrootd.cern.ch/sw/releases/RPM-GPG-KEY.txt\n' >> /etc/yum.repos.d/xrootd-testing.repo + - echo -e '[xrootd-stable]\nname=XRootD Stable repository\nbaseurl=http://xrootd.org/binaries/stable/slc/7/$basearch http://xrootd.cern.ch/sw/repos/stable/slc/7/$basearch\ngpgcheck=1\nenabled=1\nprotect=0\ngpgkey=http://xrootd.cern.ch/sw/releases/RPM-GPG-KEY.txt\n' >> /etc/yum.repos.d/xrootd-stable.repo + - yum-builddep --setopt=cern*.exclude=xrootd* --nogpgcheck -y *.src.rpm + - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" -D "dist .el7" *.src.rpm + - repo=/eos/project/s/storage-ci/www/xrootd/ceph-release/cc-7/x86_64/ + - sudo -u stci -H mkdir -p $repo + - sudo -u stci -H cp *.src.rpm $repo + - sudo -u stci -H cp RPMS/* $repo + - sudo -u stci -H createrepo --update -q $repo + tags: + - docker_node + only: + - tags + except: + - schedules + +weekly:cc7:ceph: + stage: build:rpm + image: gitlab-registry.cern.ch/linuxsupport/cc7-base + script: + - yum install --nogpg -y cmake3 make gcc-c++ rpm-build which git yum-plugin-priorities sssd-client sudo createrepo + - cd packaging/ + - echo -e '[ceph]\nname=ceph\nbaseurl=http://linuxsoft.cern.ch/mirror/download.ceph.com/rpm-nautilus/el7/x86_64/\npriority=4\ngpgcheck=0\nenabled=1\n' >> /etc/yum.repos.d/ceph.repo + - echo -e '[xrootd-experimental]\nname=XRootD Experimental repository\nbaseurl=http://storage-ci.web.cern.ch/storage-ci/xrootd/experimental/epel-7/$basearch\ngpgcheck=1\nenabled=1\nprotect=0\n' >> /etc/yum.repos.d/xrootd-experimental.repo + - yum clean all + - version=$(yum info xrootd-devel | grep Version | cut -d':' -f2 | tr -d "[:blank:]") + - release=$(yum info xrootd-devel | grep Release | cut -d':' -f2 | tr -d "[:blank:]") + - release=${release%.el7.cern} + - ./makesrpm.sh --version "$version-$release" + - yum-builddep --setopt=cern*.exclude=xrootd* --nogpgcheck -y *.src.rpm + - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm + tags: + - docker_node + only: + - schedules + +build:cc7:ceph: + stage: build:rpm + image: gitlab-registry.cern.ch/linuxsupport/cc7-base + script: + - yum install --nogpg -y cmake3 make gcc-c++ rpm-build which git yum-plugin-priorities sssd-client sudo createrepo + - cd packaging/ + - echo -e '[ceph]\nname=ceph\nbaseurl=http://linuxsoft.cern.ch/mirror/download.ceph.com/rpm-nautilus/el7/x86_64/\npriority=4\ngpgcheck=0\nenabled=1\n' >> /etc/yum.repos.d/ceph.repo + - echo -e '[xrootd-experimental]\nname=XRootD Experimental repository\nbaseurl=http://storage-ci.web.cern.ch/storage-ci/xrootd/experimental/epel-7/$basearch\ngpgcheck=1\nenabled=1\nprotect=0\n' >> /etc/yum.repos.d/xrootd-experimental.repo + - yum clean all + - version=$(yum info xrootd-devel | grep Version | cut -d':' -f2 | tr -d "[:blank:]") + - release=$(yum info xrootd-devel | grep Release | cut -d':' -f2 | tr -d "[:blank:]") + - release=${release%.el7.cern} + - ./makesrpm.sh --version "$version-$release" + - yum-builddep --setopt=cern*.exclude=xrootd* --nogpgcheck -y *.src.rpm + - rpmbuild --rebuild --define "_rpmdir RPMS/" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" *.src.rpm + - path=/eos/project/s/storage-ci/www/xrootd/ceph/cc-7/x86_64/$(date +'%Y%m%d') + - sudo -u stci -H mkdir -p $path; + - sudo -u stci -H find ${path} -type f -name '*.rpm' -delete; + - sudo -u stci -H cp RPMS/* $path; + - sudo -u stci -H createrepo --update -q $path; + tags: + - docker_node + only: + - master + except: + - tags + diff --git a/src/XrdCeph/.travis.yml b/src/XrdCeph/.travis.yml new file mode 100644 index 00000000000..0f57d7cb54e --- /dev/null +++ b/src/XrdCeph/.travis.yml @@ -0,0 +1,22 @@ +sudo: false +dist: trusty +addons: + apt: + packages: + - libxml2-dev + - libcppunit-dev +language: cpp +compiler: + - clang + - gcc +script: + - mkdir build + - pushd build + - cmake -DCMAKE_INSTALL_PREFIX=$HOME/xrootd -DENABLE_TESTS=1 .. + - make + - make install + - popd +#after_script: +# - pushd build +# - ./tests/common/text-runner ./tests/XrdClTests/libXrdClTests.so 'All Tests' +# - popd diff --git a/src/XrdCeph/CMakeLists.txt b/src/XrdCeph/CMakeLists.txt new file mode 100644 index 00000000000..c8730143079 --- /dev/null +++ b/src/XrdCeph/CMakeLists.txt @@ -0,0 +1,62 @@ +#------------------------------------------------------------------------------- +# Project description +#------------------------------------------------------------------------------- +cmake_minimum_required(VERSION 3.16...3.25) + +project( xrootd-ceph ) + +set( CMAKE_MODULE_PATH + ${PROJECT_SOURCE_DIR}/src + ${PROJECT_SOURCE_DIR}/cmake ) + +if( NOT XRDCEPH_SUBMODULE ) + if(NOT (CMAKE_VERSION VERSION_LESS "3.1")) + cmake_policy(SET CMP0054 OLD) + endif() +endif() + +include( XRootDUtils ) +CheckBuildDirectory() + +include( XRootDOSDefs ) +include( XRootDDefaults ) +include( XRootDFindLibs ) + +add_definitions( -DXRDPLUGIN_SOVERSION="${PLUGIN_VERSION}" ) + +#------------------------------------------------------------------------------- +# Generate the version header +#------------------------------------------------------------------------------- +if( NOT XRDCEPH_SUBMODULE ) + execute_process( + COMMAND ${CMAKE_SOURCE_DIR}/genversion.sh --print-only ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE XROOTD_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE ) + + add_custom_target( + XrdVersion.hh + ${CMAKE_SOURCE_DIR}/genversion.sh ${CMAKE_SOURCE_DIR} ) + + # sigh, yet another ugly hack :( + macro( add_library _target ) + _add_library( ${_target} ${ARGN} ) + add_dependencies( ${_target} XrdVersion.hh ) + endmacro() + + macro( add_executable _target ) + _add_executable( ${_target} ${ARGN} ) + add_dependencies( ${_target} XrdVersion.hh ) + endmacro() +endif() + +#------------------------------------------------------------------------------- +# Build in subdirectories +#------------------------------------------------------------------------------- +add_subdirectory( src ) + +if( BUILD_TESTS ) + ENABLE_TESTING() + add_subdirectory( tests ) +endif() + +include( XRootDSummary ) diff --git a/src/XrdCeph/COPYING b/src/XrdCeph/COPYING new file mode 100644 index 00000000000..94a9ed024d3 --- /dev/null +++ b/src/XrdCeph/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/src/XrdCeph/COPYING.BSD b/src/XrdCeph/COPYING.BSD new file mode 100644 index 00000000000..ea8ff31a74e --- /dev/null +++ b/src/XrdCeph/COPYING.BSD @@ -0,0 +1,35 @@ +******************************************************************************** +*Prior to September 2nd, 2012 the XRootD software suite was licensed under a * +*modified BSD license shown below. This applies to all code that was in the * +*XRootD git repository prior to that date. All code is now licensed under LGPL.* +* See files LICENSE, COPYING.LGPL, and COPYING for license details. * +******************************************************************************** + +Copyright (c) 2005-2012, Board of Trustees of the Leland Stanford, Jr. University. +Produced under contract DE-AC02-76-SF00515 with the US Department of Energy. +All rights reserved. + Conditions of Use +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +a. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +b. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +c. Neither the name of the Leland Stanford, Jr. University nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. +d. Products derived from this software that do not adhere to the xrootd or cmsd + protocol specifications may not use the acronyms 'cmsd', 'Scalla', 'xroot', + and 'xrootd', regardless of capitalization, to describe such derivative works. + DISCLAIMER +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/XrdCeph/COPYING.LGPL b/src/XrdCeph/COPYING.LGPL new file mode 100644 index 00000000000..65c5ca88a67 --- /dev/null +++ b/src/XrdCeph/COPYING.LGPL @@ -0,0 +1,165 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/src/XrdCeph/Doxyfile b/src/XrdCeph/Doxyfile new file mode 100644 index 00000000000..0531e916456 --- /dev/null +++ b/src/XrdCeph/Doxyfile @@ -0,0 +1,316 @@ +# Doxyfile 1.3.7 + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- +PROJECT_NAME = xrootd +PROJECT_NUMBER = +OUTPUT_DIRECTORY = doxydoc +CREATE_SUBDIRS = NO +OUTPUT_LANGUAGE = English +USE_WINDOWS_ENCODING = NO +BRIEF_MEMBER_DESC = YES +REPEAT_BRIEF = YES +ABBREVIATE_BRIEF = +ALWAYS_DETAILED_SEC = NO +INLINE_INHERITED_MEMB = NO +FULL_PATH_NAMES = NO +STRIP_FROM_PATH = +STRIP_FROM_INC_PATH = +SHORT_NAMES = NO +JAVADOC_AUTOBRIEF = NO +MULTILINE_CPP_IS_BRIEF = NO +DETAILS_AT_TOP = NO +INHERIT_DOCS = YES +DISTRIBUTE_GROUP_DOC = NO +TAB_SIZE = 8 +ALIASES = +OPTIMIZE_OUTPUT_FOR_C = NO +OPTIMIZE_OUTPUT_JAVA = NO +SUBGROUPING = YES +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- +EXTRACT_ALL = YES +EXTRACT_PRIVATE = YES +EXTRACT_STATIC = YES +EXTRACT_LOCAL_CLASSES = YES +EXTRACT_LOCAL_METHODS = NO +HIDE_UNDOC_MEMBERS = NO +HIDE_UNDOC_CLASSES = NO +HIDE_FRIEND_COMPOUNDS = NO +HIDE_IN_BODY_DOCS = NO +INTERNAL_DOCS = NO +CASE_SENSE_NAMES = YES +HIDE_SCOPE_NAMES = NO +SHOW_INCLUDE_FILES = YES +INLINE_INFO = YES +SORT_MEMBER_DOCS = YES +SORT_BRIEF_DOCS = NO +SORT_BY_SCOPE_NAME = NO +GENERATE_TODOLIST = YES +GENERATE_TESTLIST = YES +GENERATE_BUGLIST = YES +GENERATE_DEPRECATEDLIST= YES +ENABLED_SECTIONS = +MAX_INITIALIZER_LINES = 30 +SHOW_USED_FILES = YES +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- +QUIET = NO +WARNINGS = YES +WARN_IF_UNDOCUMENTED = YES +WARN_IF_DOC_ERROR = YES +WARN_FORMAT = "$file:$line: $text" +WARN_LOGFILE = +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- +INPUT = \ +src/XrdSec/XrdSecEntity.hh \ +src/XrdSec/XrdSecInterface.hh \ +src/Xrd/XrdJob.hh \ +src/Xrd/XrdBuffer.hh \ +src/Xrd/XrdScheduler.hh \ +src/Xrd/XrdLink.hh \ +src/Xrd/XrdLinkMatch.hh \ +src/Xrd/XrdProtocol.hh \ +src/XrdXrootd/XrdXrootdMonData.hh \ +src/XrdVersionPlugin.hh \ +src/XrdCks/XrdCksData.hh \ +src/XrdCks/XrdCksManager.hh \ +src/XrdCks/XrdCksCalc.hh \ +src/XrdCks/XrdCks.hh \ +src/XrdSfs/XrdSfsInterface.hh \ +src/XrdSfs/XrdSfsAio.hh \ +src/XrdNet/XrdNet.hh \ +src/XrdNet/XrdNetCmsNotify.hh \ +src/XrdNet/XrdNetConnect.hh \ +src/XrdNet/XrdNetOpts.hh \ +src/XrdNet/XrdNetSocket.hh \ +src/XrdSys/XrdSysError.hh \ +src/XrdSys/XrdSysPlatform.hh \ +src/XrdSys/XrdSysLogger.hh \ +src/XrdSys/XrdSysPthread.hh \ +src/XrdSys/XrdSysTimer.hh \ +src/XrdSys/XrdSysHeaders.hh \ +src/XrdSys/XrdSysDNS.hh \ +src/XrdSys/XrdSysXSLock.hh \ +src/XrdSys/XrdSysIOEvents.hh \ +src/XrdSys/XrdSysAtomics.hh \ +src/XrdSys/XrdSysPlugin.hh \ +src/XrdSys/XrdSysSemWait.hh \ +src/XrdClient/XrdClientConst.hh \ +src/XrdClient/XrdClientVector.hh \ +src/XrdClient/XrdClientAbs.hh \ +src/XrdClient/XrdClientAbsMonIntf.hh \ +src/XrdClient/XrdClient.hh \ +src/XrdClient/XrdClientUnsolMsg.hh \ +src/XrdClient/XrdClientAdmin.hh \ +src/XrdClient/XrdClientUrlSet.hh \ +src/XrdClient/XrdClientUrlInfo.hh \ +src/XrdClient/XrdClientEnv.hh \ +src/XrdOuc/XrdOucRash.hh \ +src/XrdOuc/XrdOucStream.hh \ +src/XrdOuc/XrdOuca2x.hh \ +src/XrdOuc/XrdOucTrace.hh \ +src/XrdOuc/XrdOucCRC.hh \ +src/XrdOuc/XrdOucErrInfo.hh \ +src/XrdOuc/XrdOucDLlist.hh \ +src/XrdOuc/XrdOucCache.hh \ +src/XrdOuc/XrdOucTList.hh \ +src/XrdOuc/XrdOucName2Name.hh \ +src/XrdOuc/XrdOucTable.hh \ +src/XrdOuc/XrdOucIOVec.hh \ +src/XrdOuc/XrdOucCallBack.hh \ +src/XrdOuc/XrdOucEnum.hh \ +src/XrdOuc/XrdOucLock.hh \ +src/XrdOuc/XrdOucTokenizer.hh \ +src/XrdOuc/XrdOucEnv.hh \ +src/XrdOuc/XrdOucString.hh \ +src/XrdOuc/XrdOucChain.hh \ +src/XrdOuc/XrdOucUtils.hh \ +src/XrdOuc/XrdOucHash.hh \ +src/XrdOss/XrdOss.hh \ +src/XrdOss/XrdOssStatInfo.hh \ +src/XrdOss/XrdOssDefaultSS.hh \ +src/XrdPosix/XrdPosixXrootd.hh \ +src/XrdPosix/XrdPosixExtern.hh \ +src/XrdPosix/XrdPosixXrootdPath.hh \ +src/XrdPosix/XrdPosixOsDep.hh \ +src/XrdPosix/XrdPosixCallBack.hh \ +src/XrdAcc/XrdAccAuthorize.hh \ +src/XrdAcc/XrdAccPrivs.hh \ +src/XProtocol/XPtypes.hh \ +src/XProtocol/XProtocol.hh \ +src/XrdCms/XrdCmsClient.hh \ +src/XrdCl/XrdClEnv.hh \ +src/XrdCl/XrdClPostMaster.hh \ +src/XrdCl/XrdClFileSystem.hh \ +src/XrdCl/XrdClPostMasterInterfaces.hh \ +src/XrdCl/XrdClBuffer.hh \ +src/XrdCl/XrdClConstants.hh \ +src/XrdCl/XrdClCopyProcess.hh \ +src/XrdCl/XrdClDefaultEnv.hh \ +src/XrdCl/XrdClMessage.hh \ +src/XrdCl/XrdClMonitor.hh \ +src/XrdCl/XrdClStatus.hh \ +src/XrdCl/XrdClTransportManager.hh \ +src/XrdCl/XrdClURL.hh \ +src/XrdCl/XrdClAnyObject.hh \ +src/XrdCl/XrdClXRootDResponses.hh \ +src/XrdCl/XrdClFile.hh \ +src/XrdFileCache/XrdFileCacheDecision.hh +src/XrdFileCache/XrdFileCacheFile.hh +src/XrdFileCache/XrdFileCache.hh +src/XrdFileCache/XrdFileCacheInfo.hh +src/XrdFileCache/XrdFileCacheIOEntireFile.hh +src/XrdFileCache/XrdFileCacheIOFileBlock.hh +src/XrdFileCache/XrdFileCacheIO.hh +src/XrdFileCache/XrdFileCachePrint.hh +src/XrdFileCache/XrdFileCacheStats.hh +src/XrdFileCache/XrdFileCacheTrace.hh + +FILE_PATTERNS = *.hh +RECURSIVE = YES +EXCLUDE = +EXCLUDE_SYMLINKS = NO +EXCLUDE_PATTERNS = +EXAMPLE_PATH = +EXAMPLE_PATTERNS = +EXAMPLE_RECURSIVE = NO +IMAGE_PATH = +INPUT_FILTER = +FILTER_SOURCE_FILES = NO +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- +SOURCE_BROWSER = NO +INLINE_SOURCES = NO +STRIP_CODE_COMMENTS = YES +REFERENCED_BY_RELATION = YES +REFERENCES_RELATION = YES +VERBATIM_HEADERS = YES +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- +ALPHABETICAL_INDEX = NO +COLS_IN_ALPHA_INDEX = 5 +IGNORE_PREFIX = +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- +GENERATE_HTML = YES +HTML_OUTPUT = html +HTML_FILE_EXTENSION = .html +HTML_HEADER = +HTML_FOOTER = +HTML_STYLESHEET = +HTML_ALIGN_MEMBERS = YES +GENERATE_HTMLHELP = NO +CHM_FILE = +HHC_LOCATION = +GENERATE_CHI = NO +BINARY_TOC = NO +TOC_EXPAND = NO +DISABLE_INDEX = NO +ENUM_VALUES_PER_LINE = 4 +GENERATE_TREEVIEW = NO +TREEVIEW_WIDTH = 250 +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- +GENERATE_LATEX = NO +LATEX_OUTPUT = latex +LATEX_CMD_NAME = latex +MAKEINDEX_CMD_NAME = makeindex +COMPACT_LATEX = NO +PAPER_TYPE = a4wide +EXTRA_PACKAGES = +LATEX_HEADER = +PDF_HYPERLINKS = NO +USE_PDFLATEX = NO +LATEX_BATCHMODE = NO +LATEX_HIDE_INDICES = NO +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- +GENERATE_RTF = NO +RTF_OUTPUT = rtf +COMPACT_RTF = NO +RTF_HYPERLINKS = NO +RTF_STYLESHEET_FILE = +RTF_EXTENSIONS_FILE = +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- +GENERATE_MAN = NO +MAN_OUTPUT = man +MAN_EXTENSION = .3 +MAN_LINKS = NO +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- +GENERATE_XML = NO +XML_OUTPUT = xml +XML_SCHEMA = +XML_DTD = +XML_PROGRAMLISTING = YES +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- +GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- +GENERATE_PERLMOD = NO +PERLMOD_LATEX = NO +PERLMOD_PRETTY = YES +PERLMOD_MAKEVAR_PREFIX = +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- +ENABLE_PREPROCESSING = YES +MACRO_EXPANSION = NO +EXPAND_ONLY_PREDEF = NO +SEARCH_INCLUDES = YES +INCLUDE_PATH = +INCLUDE_FILE_PATTERNS = +PREDEFINED = +EXPAND_AS_DEFINED = +SKIP_FUNCTION_MACROS = YES +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- +TAGFILES = +GENERATE_TAGFILE = +ALLEXTERNALS = NO +EXTERNAL_GROUPS = YES +PERL_PATH = /usr/bin/perl +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- +CLASS_DIAGRAMS = YES +HIDE_UNDOC_RELATIONS = YES +HAVE_DOT = YES +CLASS_GRAPH = YES +COLLABORATION_GRAPH = YES +UML_LOOK = NO +TEMPLATE_RELATIONS = NO +INCLUDE_GRAPH = YES +INCLUDED_BY_GRAPH = YES +CALL_GRAPH = NO +GRAPHICAL_HIERARCHY = YES +DOT_IMAGE_FORMAT = png +DOT_PATH = +DOTFILE_DIRS = +MAX_DOT_GRAPH_WIDTH = 1024 +MAX_DOT_GRAPH_HEIGHT = 1024 +MAX_DOT_GRAPH_DEPTH = 0 +GENERATE_LEGEND = YES +DOT_CLEANUP = YES +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- +SEARCHENGINE = NO diff --git a/src/XrdCeph/LICENSE b/src/XrdCeph/LICENSE new file mode 100644 index 00000000000..8fd5621be2c --- /dev/null +++ b/src/XrdCeph/LICENSE @@ -0,0 +1,18 @@ +"Copyright (c) 2005-2012, Board of Trustees of the Leland Stanford, Jr. University.\n" +"Produced under contract DE-AC02-76-SF00515 with the US Department of Energy. \n" +"All rights reserved. The copyright holder's institutional names may not be used to\n" +"endorse or promote products derived from this software without specific prior \n" +"written permission.\n\n" +"This file is part of the XRootD software suite. \n\n" +"XRootD is free software: you can redistribute it and/or modify it under the terms \n" +"of the GNU Lesser General Public License as published by the Free Software \n" +"Foundation, either version 3 of the License, or (at your option) any later version.\n\n" +"XRootD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n" +"without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR \n" +"PURPOSE. See the GNU Lesser General Public License for more details. \nn" +"You should have received a copy of the GNU Lesser General Public License along \n" +"with XRootD in a file called COPYING.LESSER (LGPL license) and file COPYING (GPL \n" +"license). If not, see .\n\n" +"Prior to September 2nd, 2012 the XRootD software suite was licensed under a\n" +"modified BSD license (see file COPYING.BSD). This applies to all code that\n" +"was in the XRootD git repository prior to that date.\n" diff --git a/src/XrdCeph/README b/src/XrdCeph/README new file mode 100644 index 00000000000..e2bf17c3df6 --- /dev/null +++ b/src/XrdCeph/README @@ -0,0 +1,54 @@ + +-------------------------------------------------------------------------------- + _ _ ______ _____ + \ \ / (_____ \ _ (____ \ + \ \/ / _____) ) ___ ___ | |_ _ \ \ + ) ( (_____ ( / _ \ / _ \| _)| | | | + / /\ \ | | |_| | |_| | |__| |__/ / + /_/ \_\ |_|\___/ \___/ \___)_____/ + +-------------------------------------------------------------------------------- + +0. xrootd-ceph is a OSS layer XRootD plug-in for interfacing with Ceph storage + platform. The plug-in has to be build against respective Ceph version, the + repository can be found at: + + https://download.ceph.com/rpm-{ceph-release}/{distro}/$basearch + +1. S U P P O R T E D O P E R A T I N G S Y S T E M S + + XRootD is supported on the following platforms: + + * RedHat Enterprise Linux 7 and derivatives (Scientific Linux) + compiled with gcc + +2. B U I L D I N S T R U C T I O N S + +2.1 Build system + + xrootd-ceph uses CMake to handle the build process. Please use CMake version 3 or greater (e.g. cmake3). + +2.2 Build steps + + * Create an empty build directory: + + mkdir build + cd build + + * Ensure that the correct plugin version number is set in cmake/XRootDDefaults.cmake: + + if( NOT XRDCEPH_SUBMODULE ) + define_default( PLUGIN_VERSION 5 ) + endif() + + * Generate the build system files using cmake, ie: + + cmake /path/to/the/xrootd-ceph/source -DCMAKE_INSTALL_PREFIX=/opt/xrootd + + * Build the source: + + make + + * Install the shared libraries: + + make install diff --git a/VERSION_INFO b/src/XrdCeph/VERSION_INFO similarity index 100% rename from VERSION_INFO rename to src/XrdCeph/VERSION_INFO diff --git a/src/XrdCeph/cmake/FindCppUnit.cmake b/src/XrdCeph/cmake/FindCppUnit.cmake new file mode 100644 index 00000000000..cc6e7400191 --- /dev/null +++ b/src/XrdCeph/cmake/FindCppUnit.cmake @@ -0,0 +1,30 @@ +# Try to find CPPUnit +# Once done, this will define +# +# CPPUNIT_FOUND - system has cppunit +# CPPUNIT_INCLUDE_DIRS - the cppunit include directories +# CPPUNIT_LIBRARIES - cppunit libraries directories + +find_path( CPPUNIT_INCLUDE_DIRS cppunit/ui/text/TestRunner.h + HINTS + ${CPPUNIT_DIR} + $ENV{CPPUNIT_DIR} + /usr + /opt + PATH_SUFFIXES include +) + +find_library( CPPUNIT_LIBRARIES cppunit + HINTS + ${CPPUNIT_DIR} + $ENV{CPPUNIT_DIR} + /usr + /opt + PATH_SUFFIXES lib +) + +set(CPPUNIT_INCLUDE_DIRS ${CPPUNIT_INCLUDE_DIR}) +set(CPPUNIT_LIBRARIES ${CPPUNIT_LIBRARY}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(CppUnit DEFAULT_MSG CPPUNIT_INCLUDE_DIRS CPPUNIT_LIBRARIES) diff --git a/src/XrdCeph/cmake/FindXRootD.cmake b/src/XrdCeph/cmake/FindXRootD.cmake new file mode 100644 index 00000000000..a3596ebc338 --- /dev/null +++ b/src/XrdCeph/cmake/FindXRootD.cmake @@ -0,0 +1,35 @@ +# Try to find XRootD +# Once done, this will define +# +# XROOTD_FOUND - system has xrootd +# XROOTD_INCLUDE_DIRS - the xrootd include directories +# XROOTD_LIBRARIES - xrootd libraries directories + +if( XRDCEPH_SUBMODULE ) + set( XROOTD_INCLUDE_DIRS ${CMAKE_SOURCE_DIR}/src ) + set( XROOTD_LIBRARIES XrdUtils ) +else() + find_path( XROOTD_INCLUDE_DIRS XrdSfs/XrdSfsAio.hh + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt + PATH_SUFFIXES include/xrootd + ) + + find_library( XROOTD_LIBRARIES XrdUtils + HINTS + ${XROOTD_DIR} + $ENV{XROOTD_DIR} + /usr + /opt + PATH_SUFFIXES lib + ) +endif() + +set(XROOTD_INCLUDE_DIR ${XROOTD_INCLUDE_DIRS}) +set(XROOTD_LIBRARY ${XROOTD_LIBRARIES}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(XRootD DEFAULT_MSG XROOTD_INCLUDE_DIRS XROOTD_LIBRARIES) diff --git a/src/XrdCeph/cmake/Findceph.cmake b/src/XrdCeph/cmake/Findceph.cmake new file mode 100644 index 00000000000..661891de850 --- /dev/null +++ b/src/XrdCeph/cmake/Findceph.cmake @@ -0,0 +1,43 @@ +# - Find ceph +# +# RADOS_INCLUDE_DIR - location of the ceph-devel header files for rados +# RADOS_LIBS - list of rados libraries, with full path +# RADOS_FOUND + +find_path( + RADOS_INCLUDE_DIR + radosstriper/libradosstriper.hpp + HINTS + ${CEPH_DIR} + $ENV{CEPH_DIR} + /usr + /opt + PATH_SUFFIXES include +) + +find_library( + RADOSSTRIPER_LIB + NAMES radosstriper + HINTS + ${CEPH_DIR} + $ENV{CEPH_DIR} + /usr + /opt + PATH_SUFFIXES lib +) + +find_library( + RADOS_LIB + NAMES rados + HINTS + ${CEPH_DIR} + $ENV{CEPH_DIR} + /usr + /opt + PATH_SUFFIXES lib +) + +set(RADOS_LIBS ${RADOS_LIB} ${RADOSSTRIPER_LIB}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(ceph DEFAULT_MSG RADOS_INCLUDE_DIR RADOS_LIBS) diff --git a/src/XrdCeph/cmake/GNUInstallDirs.cmake b/src/XrdCeph/cmake/GNUInstallDirs.cmake new file mode 100644 index 00000000000..a114dcb2e18 --- /dev/null +++ b/src/XrdCeph/cmake/GNUInstallDirs.cmake @@ -0,0 +1,182 @@ +# - Define GNU standard installation directories +# Provides install directory variables as defined for GNU software: +# http://www.gnu.org/prep/standards/html_node/Directory-Variables.html +# Inclusion of this module defines the following variables: +# CMAKE_INSTALL_ - destination for files of a given type +# CMAKE_INSTALL_FULL_ - corresponding absolute path +# where is one of: +# BINDIR - user executables (bin) +# SBINDIR - system admin executables (sbin) +# LIBEXECDIR - program executables (libexec) +# SYSCONFDIR - read-only single-machine data (etc) +# SHAREDSTATEDIR - modifiable architecture-independent data (com) +# LOCALSTATEDIR - modifiable single-machine data (var) +# LIBDIR - object code libraries (lib or lib64) +# INCLUDEDIR - C header files (include) +# OLDINCLUDEDIR - C header files for non-gcc (/usr/include) +# DATAROOTDIR - read-only architecture-independent data root (share) +# DATADIR - read-only architecture-independent data (DATAROOTDIR) +# INFODIR - info documentation (DATAROOTDIR/info) +# LOCALEDIR - locale-dependent data (DATAROOTDIR/locale) +# MANDIR - man documentation (DATAROOTDIR/man) +# DOCDIR - documentation root (DATAROOTDIR/doc/PROJECT_NAME) +# Each CMAKE_INSTALL_ value may be passed to the DESTINATION options of +# install() commands for the corresponding file type. If the includer does +# not define a value the above-shown default will be used and the value will +# appear in the cache for editing by the user. +# Each CMAKE_INSTALL_FULL_ value contains an absolute path constructed +# from the corresponding destination by prepending (if necessary) the value +# of CMAKE_INSTALL_PREFIX. + +#============================================================================= +# Copyright 2011 Nikita Krupen'ko +# Copyright 2011 Kitware, Inc. +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + +# Installation directories +# +if(NOT DEFINED CMAKE_INSTALL_BINDIR) + set(CMAKE_INSTALL_BINDIR "bin" CACHE PATH "user executables (bin)") +endif() + +if(NOT DEFINED CMAKE_INSTALL_SBINDIR) + set(CMAKE_INSTALL_SBINDIR "sbin" CACHE PATH "system admin executables (sbin)") +endif() + +if(NOT DEFINED CMAKE_INSTALL_LIBEXECDIR) + set(CMAKE_INSTALL_LIBEXECDIR "libexec" CACHE PATH "program executables (libexec)") +endif() + +if(NOT DEFINED CMAKE_INSTALL_SYSCONFDIR) + set(CMAKE_INSTALL_SYSCONFDIR "etc" CACHE PATH "read-only single-machine data (etc)") +endif() + +if(NOT DEFINED CMAKE_INSTALL_SHAREDSTATEDIR) + set(CMAKE_INSTALL_SHAREDSTATEDIR "com" CACHE PATH "modifiable architecture-independent data (com)") +endif() + +if(NOT DEFINED CMAKE_INSTALL_LOCALSTATEDIR) + set(CMAKE_INSTALL_LOCALSTATEDIR "var" CACHE PATH "modifiable single-machine data (var)") +endif() + +if(NOT DEFINED CMAKE_INSTALL_LIBDIR) + set(_LIBDIR_DEFAULT "lib") + # Override this default 'lib' with 'lib64' iff: + # - we are on Linux system but NOT cross-compiling + # - we are NOT on debian + # - we are on a 64 bits system + # reason is: amd64 ABI: http://www.x86-64.org/documentation/abi.pdf + # Note that the future of multi-arch handling may be even + # more complicated than that: http://wiki.debian.org/Multiarch + if(CMAKE_SYSTEM_NAME MATCHES "Linux" + AND NOT CMAKE_CROSSCOMPILING + AND NOT EXISTS "/etc/debian_version") + if(NOT DEFINED CMAKE_SIZEOF_VOID_P) + message(AUTHOR_WARNING + "Unable to determine default CMAKE_INSTALL_LIBDIR directory because no target architecture is known. " + "Please enable at least one language before including GNUInstallDirs.") + else() + if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + set(_LIBDIR_DEFAULT "lib64") + endif() + endif() + endif() + set(CMAKE_INSTALL_LIBDIR "${_LIBDIR_DEFAULT}" CACHE PATH "object code libraries (${_LIBDIR_DEFAULT})") +endif() + +if(NOT DEFINED CMAKE_INSTALL_INCLUDEDIR) + set(CMAKE_INSTALL_INCLUDEDIR "include" CACHE PATH "C header files (include)") +endif() + +if(NOT DEFINED CMAKE_INSTALL_OLDINCLUDEDIR) + set(CMAKE_INSTALL_OLDINCLUDEDIR "/usr/include" CACHE PATH "C header files for non-gcc (/usr/include)") +endif() + +if(NOT DEFINED CMAKE_INSTALL_DATAROOTDIR) + set(CMAKE_INSTALL_DATAROOTDIR "share" CACHE PATH "read-only architecture-independent data root (share)") +endif() + +#----------------------------------------------------------------------------- +# Values whose defaults are relative to DATAROOTDIR. Store empty values in +# the cache and store the defaults in local variables if the cache values are +# not set explicitly. This auto-updates the defaults as DATAROOTDIR changes. + +if(NOT CMAKE_INSTALL_DATADIR) + set(CMAKE_INSTALL_DATADIR "" CACHE PATH "read-only architecture-independent data (DATAROOTDIR)") + set(CMAKE_INSTALL_DATADIR "${CMAKE_INSTALL_DATAROOTDIR}") +endif() + +if(NOT CMAKE_INSTALL_INFODIR) + set(CMAKE_INSTALL_INFODIR "" CACHE PATH "info documentation (DATAROOTDIR/info)") + set(CMAKE_INSTALL_INFODIR "${CMAKE_INSTALL_DATAROOTDIR}/info") +endif() + +if(NOT CMAKE_INSTALL_LOCALEDIR) + set(CMAKE_INSTALL_LOCALEDIR "" CACHE PATH "locale-dependent data (DATAROOTDIR/locale)") + set(CMAKE_INSTALL_LOCALEDIR "${CMAKE_INSTALL_DATAROOTDIR}/locale") +endif() + +if(NOT CMAKE_INSTALL_MANDIR) + set(CMAKE_INSTALL_MANDIR "" CACHE PATH "man documentation (DATAROOTDIR/man)") + set(CMAKE_INSTALL_MANDIR "${CMAKE_INSTALL_DATAROOTDIR}/man") +endif() + +if(NOT CMAKE_INSTALL_DOCDIR) + set(CMAKE_INSTALL_DOCDIR "" CACHE PATH "documentation root (DATAROOTDIR/doc/PROJECT_NAME)") + set(CMAKE_INSTALL_DOCDIR "${CMAKE_INSTALL_DATAROOTDIR}/doc/${PROJECT_NAME}") +endif() + +#----------------------------------------------------------------------------- + +mark_as_advanced( + CMAKE_INSTALL_BINDIR + CMAKE_INSTALL_SBINDIR + CMAKE_INSTALL_LIBEXECDIR + CMAKE_INSTALL_SYSCONFDIR + CMAKE_INSTALL_SHAREDSTATEDIR + CMAKE_INSTALL_LOCALSTATEDIR + CMAKE_INSTALL_LIBDIR + CMAKE_INSTALL_INCLUDEDIR + CMAKE_INSTALL_OLDINCLUDEDIR + CMAKE_INSTALL_DATAROOTDIR + CMAKE_INSTALL_DATADIR + CMAKE_INSTALL_INFODIR + CMAKE_INSTALL_LOCALEDIR + CMAKE_INSTALL_MANDIR + CMAKE_INSTALL_DOCDIR + ) + +# Result directories +# +foreach(dir + BINDIR + SBINDIR + LIBEXECDIR + SYSCONFDIR + SHAREDSTATEDIR + LOCALSTATEDIR + LIBDIR + INCLUDEDIR + OLDINCLUDEDIR + DATAROOTDIR + DATADIR + INFODIR + LOCALEDIR + MANDIR + DOCDIR + ) + if(NOT IS_ABSOLUTE ${CMAKE_INSTALL_${dir}}) + set(CMAKE_INSTALL_FULL_${dir} "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_${dir}}") + else() + set(CMAKE_INSTALL_FULL_${dir} "${CMAKE_INSTALL_${dir}}") + endif() +endforeach() diff --git a/src/XrdCeph/cmake/XRootDDefaults.cmake b/src/XrdCeph/cmake/XRootDDefaults.cmake new file mode 100644 index 00000000000..15416a3046d --- /dev/null +++ b/src/XrdCeph/cmake/XRootDDefaults.cmake @@ -0,0 +1,17 @@ +#------------------------------------------------------------------------------- +# Define the default build parameters +#------------------------------------------------------------------------------- +if( "${CMAKE_BUILD_TYPE}" STREQUAL "" ) + if( Solaris AND NOT SUNCC_CAN_DO_OPTS ) + set( CMAKE_BUILD_TYPE Debug ) + else() + set( CMAKE_BUILD_TYPE RelWithDebInfo ) + endif() +endif() + +if( NOT XRDCEPH_SUBMODULE ) + define_default( PLUGIN_VERSION 5 ) +endif() + +define_default( ENABLE_TESTS FALSE ) +define_default( ENABLE_CEPH TRUE ) diff --git a/src/XrdCeph/cmake/XRootDFindLibs.cmake b/src/XrdCeph/cmake/XRootDFindLibs.cmake new file mode 100644 index 00000000000..78570502ca1 --- /dev/null +++ b/src/XrdCeph/cmake/XRootDFindLibs.cmake @@ -0,0 +1,16 @@ +#------------------------------------------------------------------------------- +# Find the required libraries +#------------------------------------------------------------------------------- + +find_package( XRootD REQUIRED ) + +find_package( ceph REQUIRED ) + +if( ENABLE_TESTS ) + find_package( CppUnit ) + if( CPPUNIT_FOUND ) + set( BUILD_TESTS TRUE ) + else() + set( BUILD_TESTS FALSE ) + endif() +endif() diff --git a/src/XrdCeph/cmake/XRootDOSDefs.cmake b/src/XrdCeph/cmake/XRootDOSDefs.cmake new file mode 100644 index 00000000000..eadc24952b7 --- /dev/null +++ b/src/XrdCeph/cmake/XRootDOSDefs.cmake @@ -0,0 +1,44 @@ +#------------------------------------------------------------------------------- +# Define the OS variables +#------------------------------------------------------------------------------- + +include( CheckCXXSourceRuns ) + +add_definitions( -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 ) +set( LIBRARY_PATH_PREFIX "lib" ) + +#------------------------------------------------------------------------------- +# GCC +#------------------------------------------------------------------------------- +if( CMAKE_COMPILER_IS_GNUCXX ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11" ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror" ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter" ) + # gcc 4.1 is retarded + execute_process( COMMAND ${CMAKE_C_COMPILER} -dumpversion + OUTPUT_VARIABLE GCC_VERSION ) + if( (GCC_VERSION VERSION_GREATER 4.1 OR GCC_VERSION VERSION_EQUAL 4.1) + AND GCC_VERSION VERSION_LESS 4.2 ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-strict-aliasing" ) + endif() + + # for 4.9.3 or greater the 'omit-frame-pointer' + # interfears with custom semaphore implementation + if( (GCC_VERSION VERSION_GREATER 4.9.2) AND (USE_LIBC_SEMAPHORE EQUAL 0) ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer" ) + endif() + + # gcc 6.0 is more pedantic + if( GCC_VERSION VERSION_GREATER 6.0 OR GCC_VERSION VERSION_EQUAL 6.0 ) + set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=misleading-indentation" ) + endif() +endif() + +#------------------------------------------------------------------------------- +# Linux +#------------------------------------------------------------------------------- +set( Linux TRUE ) +include( GNUInstallDirs ) +add_definitions( -D__linux__=1 ) +set( EXTRA_LIBS rt ) + diff --git a/src/XrdCeph/cmake/XRootDSummary.cmake b/src/XrdCeph/cmake/XRootDSummary.cmake new file mode 100644 index 00000000000..d8294d37c2f --- /dev/null +++ b/src/XrdCeph/cmake/XRootDSummary.cmake @@ -0,0 +1,21 @@ +#------------------------------------------------------------------------------- +# Print the configuration summary +#------------------------------------------------------------------------------- +set( TRUE_VAR TRUE ) +component_status( CEPH TRUE_VAR CEPH_FOUND ) +component_status( XROOTD TRUE_VAR XROOTD_FOUND ) +component_status( TESTS BUILD_TESTS CPPUNIT_FOUND ) + +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) +message( STATUS "----------------------------------------" ) +message( STATUS "Installation path: " ${CMAKE_INSTALL_PREFIX} ) +message( STATUS "C Compiler: " ${CMAKE_C_COMPILER} ) +message( STATUS "C++ Compiler: " ${CMAKE_CXX_COMPILER} ) +message( STATUS "Build type: " ${CMAKE_BUILD_TYPE} ) +message( STATUS "Plug-in version: " ${PLUGIN_VERSION} ) +message( STATUS "" ) +message( STATUS "CEPH: " ${STATUS_CEPH} ) +message( STATUS "XRootD: " ${STATUS_XROOTD} ) +message( STATUS "Tests: " ${STATUS_TESTS} ) +message( STATUS "----------------------------------------" ) +endif() diff --git a/src/XrdCeph/cmake/XRootDUtils.cmake b/src/XrdCeph/cmake/XRootDUtils.cmake new file mode 100644 index 00000000000..6fbdf0fc9e7 --- /dev/null +++ b/src/XrdCeph/cmake/XRootDUtils.cmake @@ -0,0 +1,39 @@ + +#------------------------------------------------------------------------------- +# Add a compiler define flag if a variable is defined +#------------------------------------------------------------------------------- +macro( define_default variable value ) + if( NOT DEFINED ${variable} ) + set( ${variable} ${value} ) + endif() +endmacro() + +macro( component_status name flag found ) + if( ${flag} AND ${found} ) + set( STATUS_${name} "yes" ) + elseif( ${flag} AND NOT ${found} ) + set( STATUS_${name} "libs not found" ) + else() + set( STATUS_${name} "disabled" ) + endif() +endmacro() + +#------------------------------------------------------------------------------- +# Detect in source builds +#------------------------------------------------------------------------------- +function( CheckBuildDirectory ) + + # Get Real Paths of the source and binary directories + get_filename_component( srcdir "${CMAKE_SOURCE_DIR}" REALPATH ) + get_filename_component( bindir "${CMAKE_BINARY_DIR}" REALPATH ) + + # Check for in-source builds + if( ${srcdir} STREQUAL ${bindir} ) + message( FATAL_ERROR "XRootD cannot be built in-source! " + "Please run cmake outside the " + "source directory and be sure to remove " + "CMakeCache.txt or CMakeFiles if they " + "exist in the source directory." ) + endif() + +endfunction() diff --git a/src/XrdCeph/docs/PreReleaseNotes.txt b/src/XrdCeph/docs/PreReleaseNotes.txt new file mode 100644 index 00000000000..a414e1aa765 --- /dev/null +++ b/src/XrdCeph/docs/PreReleaseNotes.txt @@ -0,0 +1,8 @@ +====== +XRootD +====== + +Prerelease Notes +================ + + diff --git a/src/XrdCeph/docs/ReleaseNotes.txt b/src/XrdCeph/docs/ReleaseNotes.txt new file mode 100644 index 00000000000..c8a8ceb19dc --- /dev/null +++ b/src/XrdCeph/docs/ReleaseNotes.txt @@ -0,0 +1,1692 @@ +====== +XRootD +====== + +Release Notes +============= + + +------------- +Version 4.8.0 +------------- + ++ **New Features** + * **[XrdCl]** Local redirection and local file support. + * **[XrdCl]** merge xrdfs ls results if not unique, closes #541. + * **[XrdCl]** Provide client specific CGI info. + * **[XrdCl]** File::WriteV implementation, closes #388. + * **[XrdHttp]** Pass the HTTP verb to the external handler for path + matching. + * **[XrdHttp]** Allow one to access the XrdSecEntity object associated + with a request. + * **[XrdHttp]** Allow filtering based on HTTP verb in MatchesPath. + * **[XrdHttp]** Allow overwrites to be done on PUT. + * **[XrdHttp]** Allow multiple external handlers to be loaded by XrdHttp. + ++ **Major bug fixes** + * **[Server]** Correctly handle monEnt on file close to avoid SEGV. + Fixes #618. + * **[Server]** Poperly handle file descriptors up to 65535. + Fixes #607. + * **[Server]** Fix handling of >65K attached files (active links). + Fixes #623. + * **[Server]** Make sure doPost does not become <0 (regression introduced + in 4.7.1). + * **[Proxy]** Avoid SEGV when localroot specified w/o remote root. + Fixes #627. + * **[XrdCl]** Connection Window should be applied per IP address. + Fixes #625. + * **[XrdCl]** Write request and raw data with single writev, fixes #609. + * **[XrdHttp]** Allow XrdSfsGetDefaultFileSystem to be called multiple + times. + * **[XrdHttp]** Correct external handling logic. + * **[XrdSecgsi]** Use stack for proper cleaning of invalidated CRLs and CAs. + ++ **Minor bug fixes** + * **[Server]** Print error msg and close socket when a FD cannot. + be handled. + * **[Server]** Close additional loophole for fstream disconnect. + * **[Server]** Always unhook the statistcs object from xfr monitoring + if hooked. + * **[Server]** Ruggedize TPC to be less sensitive to protocol violations. + * **[Server]** Correct tpc directive scanning and make it more obvious. + Fixes #604. + * **[Server]** Enable url rewrites. Eliminates GSI roadblock. + * **[Server]** Do not reference a deleted object. + * **[XrdSsi]** Make sure to finalyze all requests upon disc, fixes #616. + * **[XrdHttp]** Handle properly http.secretkey. + * **[XrdCl]** various memory releated fixes. + * **[XrdPy]** Translate binary buffers into bytes objects, closes #632 + ++ **Miscellaneous** + * **[RPM]** Add python3 sub package. + * **[RPM]** Rename python sub-package, closes #614. + * **[Py]** Facilitate building python bindings with wheel. + +------------- +Version 4.7.1 +------------- + ++ **Major bug fixes** + * **[XrdSecgsi]** Fix segv in cache checking, fixes #595 + * **[XrdHttp]** Fix for the persistent connection issue. + * **[XrdHttp]** Fix FD leak when a socket error is encountered. + * **[XrdSsi]** Avoid race condition when response is posted. + * **[XrdSsi]** Avoid state conflict when request is being processed and + client asks for response. + * **[XrdCl]** Prevent segv in case user has no name. + * **[Server]** Close link on enable errors to prevent socket leaks. + ++ **Minor bug fixes** + * **[XrdLink]** Increment the IOSemaphore once for each waiting thread. + * **[XrdHttp]** Make sure that the preexisting url tokens are properly + quoted when generating a redirection. + * **[XrdCl]** Fix invalid memory reads/writes when RAII finalizes mutex + after the object has been deleted. + ++ **Miscellaneous** + * **[XrdCl]** Log last error in case redirect limit has been reached. + * **[XrdCl]** Add option to read credentials under different fsuid/fsgid. + * **[XrdCl]** Accept empty login response for protocol <= 2.8.9 + (This is only to ensure compatibility with dCache, which + due to its inaccurate implementation of XRoot protocol in + some cases returns an empty login response for protocol + version <= 2.8.9.) + * **[XrdCl]** Add envar to config Nagle algorithm. + * **[XrdSsi]** Reinitializ response object after Finished() so it can + reused. +* **[XrdHttp]** Header2cgi directive. + +------------- +Version 4.7.0 +------------- + ++ **New Features** + * **[Proxy]** Make cache I/O synchronization tunable. + * **[Proxy]** Allow caching of S3-style objects. + * **[Proxy/Posix]** Allow Name2Name to populate cache using the LFN. + * **[Posix]** Enable LITE feature in Posix preload library. + * **[Posix]** Implement serverless file caching (disk or memory). + * **[Server]** Allow storing S3-style objects in a file system. + * **[Server]** Add xrootd.fsoverload directive to handle filesystem overloads. + * **[Server]** Allow port to be specified for a supervisor. + * **[Server]** Add org and role types to AuthDB authorization. + * **[Server]** Allow definition and test of compound authorization identifiers. + * **[Server/Packaging]** Handle systemd socket inheritance. + * **[XrdApps]** Add XrdClProxyPlugin implementation. + * **[XrdCl]** Extreme copy implementation. + * **[XrdCl]** Delegate all callbacks to the thread-pool. + * **[XrdCl]** xrdfs: add recursive list, closes #421. + * **[XrdCeph]** Added support for namelib in ceph plugin . + * **[XrdFfs]** Implement xrootdfs_create. + * **[Python]** Python 3 support in C / Python interface. + * **[XrdHttp]** Make XrdHTTP able to forward HTTP requests to an external, + optional plugin (conceptually similar to CGI). + * **[Server]** XrdSsi V2 (scalable service interface) implementation. + ++ **Major bug fixes** + * **[XrdCl]** Avoid deadlock between FSH deletion and Tick() timeout. + * **[XrdCl]** Process virtual redierections in the threadpool. + * **[Xrd] Fix handling of sendfile offset argument. + ++ **Minor bug fixes** + * **[Server]** Make file locking independent of FS plugin. Fixes #533 + * **[Server]** Correct debug message interval for free space report. + * **[XrdCeph]** Fixed internal (f)stat so that it sets S_IFREG in returned mode. + * **[XrdCeph]** properly return ENOENT when file does not exist in open for read. + * **[XrdCeph]** Fixed configuration of the XrdCephOss module. + * **[XrdCeph]** Fixed some resource leak when Posix_Open fails. + * **[XrdFfs]** Remove default fuse argument "allow_other" as it is impossible + to unset. + * **[XrdFfs]** Check file descriptor before using it in xrootdfs wcache. + * **[XrdFfs]** Add more error checks when creating write cache. + * **[XrdFfs]** Avoid using literal 1024, replace with MAXROOTURLLEN. + * **[XrdFfs]** Control allow_other by env XROOTD_NOALLOWOTHER. + * **[XrdFfs]** Rewrite xrootdfs_mknod, extract low-level function. + * **[XrdCl]** Check login resp size, fixes #530 + * **[XrdCl]** Avoid FileStateHandler deadlock while forking. + * **[XrdCl]** Handle failed stateful operations without XrdCl::File lock + being locked. + * **[XrdPosix]** Use strncpy when copying checksum. + * **[RPM]** Fix init script bad exit code, fixes #536 + * **[XrdBuffer]** Decrement total buffer count when freeing buffers. + ++ **Miscellaneous** + * **[Server]** Re-enable the oss.fdlimit directive to allow POSIX preload+xrootd. + * **[Server]** Avoid thread pile-up durin slow close operations. + * **[Proxy]** Simplify delayed destruction on wait vs post. + * **[Posix]** Convert to using universal tracing facility. + * **[CI]** Add Travis CI configuration. + * **[CI]** Add .gitlab-ci.yml for gitlab CI. + * **[Packaging]** Add a sample XrdHttp config file. + * **[Packaging]** Make RPM version configurable by the user. + * **[Packaging]** Debian packaging. + * **[RPM/CMake]** Enable c++0x/c++11 by default. + * **[Crypto] Remove unused crypto code. + * **[XrdFileCache]** Add configuration parameter for flush frequency. + * **[XrdFileCache]** Alter ram limits and blocks size parameter if caching + is on the client side. + * **[XrdSut]** New XrdSutCache based on XrdOucHash. + * **[XrdSecgsi]** do not delete explicitely the CRL in Delete. + * **[XrdSut/Crypto]** Secgsi improvements: new version of XrdSutCache, + lightweith locking (PR #539). + +------------- +Version 4.6.1 +------------- + ++ **Major bug fixes** + + * **[Server/Proxy]** Avoid SEGV when close(), closedir() returns an error. + * **[cmsd]** Fix feature interaction causing improper file existence to be sent. + * **[XrdCrypto/XrdSecgsi]** Make sure the CRL is loaded for the right CA. + * **[XrdCrypto]** Support for OpenSSL 1.1 + * **[XrdSecgsi]** do not build/package libXrdSecgsiGMAPLDAP-4.so. + * **[XrdSecgsi]** Improve detection of errors when loading CRL. + * **[XrdSecgsi]** Fix for valid legacy proxy detection (PR #469) + * **[XrdSecgsi]** Absent CRLs not an error (#465) + * **[XrdSecgsi]** Fix for CA chain verification segfault (issue #463) + * **[XrdSecgsi]** Two memory leaks (PR #503) + * **[XrdCl]** Make sure there is no request/response mismatch, when + the retry logics tries to recover from an error. + * **[XrdCl/Server]** Be case insensitive when it comes to checksum names. + * **[XrdCeph]** Fix ability to read back a file written with O_RDWR flags. + * **[XrdCeph]** Disable logging of every read and write operation. A proper + debug-level logging would be needed instead. + * **[XrdCeph]** Added statistics about read/write operations in the + close log. + ++ **Minor bug fixes** + * **[XrdHttp]** Make the XrdHttpSecXtractor API backwards compatible. + * **[XrdFileCache]** Make caching proxy configuration backwards + compatible. + * **[XrdFileCache]** Fix cache v1 to cache v2 bridge after introducing + cache v2. + * **[XrdSec]** Use CommonCrypto header instead of openssl for SHA on OSX. + * **[XrdSeckrb5]** Fix memory leaks in client context and cache. + * **[Server/Logrotate]** Make sure XRootD logrotate does not interfire with + system logrotate, fixes #490 + * ** [Server]** Avoid std::ABORT should a naked logfile path be specified. + * **[XrdCl]** Make sure ForkHandler doesn't segv if PostMaster is null, + fixes #489 + * **[Packaging]** Set the working dir to /var/spool/xrootd on CC7, + fixes #365 + * **[Packaging]** On platforms where systemd is available, manage files in + /var/run with tmpfiles.d, fixes #485 + ++ **Miscellaneous** + * **[XrdPosix]** Add new minpages option to pss.cache to support large pages. + * **[XrdPosix]** Make XrdPosix.hh a public header; closes #479 + * **[XrdApps]** Remove XrdClient dependency from xrdadler32. + * **[Server]** Add XrdCksAssist functions to help handle XRootD checksums. + * **[Server/Proxy]** Move disk sync operations out of IO::ioActive() call. + * **[Server/Proxy]** Change severity IO::initLocalStat() log message. + * **[XrdFileCache]** Ease development of decision plugins. +* **[XrdFileCache]** Use ref-counts on File objects. + +------------- +Version 4.6.0 +------------- + ++ **New Features** + * **[XrdCms]** Add non-blocking sends to avoid slow links. + * **[XrdFileCache]** File caching proxy V2 (and new pss async interface). + ++ **Major bug fixes** + * **[XrdCeph]** Account for return Ceph xattr return codes. + * **[XrdCeph]** Fixed initialization of Ceph clusters when stripers are not used. + * **[XrdCrypto]** Improved determination of X509 certificate type, + including proxy version + * **[XrdHttp]** Fix memory leak in Bridge protocol (affects HTTP). + * **[XrdSecgsi]** Several improvements in the way CRLs are checked and reloaded. + * **[XrdCl]** Protect against spurious wakeups in SyncResponseHandler. + * **[XrdCl]** On read-timeout, if the stream is broken, make sure the request and + its handler are not double deleted. + ++ **Minor bug fixes** + * **[XrdCl]** Check if the file was correctly closed upon ZipArchiveReader destruction. + * **[Server]** Add limits for prepare requests. + * **[Server]** Delete buffers when the buffer manager is deleted. Fixes #414 + * **[Server]** Do not double count overlapping spaces. Fixes #425 + * **[XrdHttp]** Allow unauthenticated https clients. + * **[XrdHttp]** Make Xrdhttp secure by default (rejecting proxy cert in the absence + of a proper SecXtractor plugin) + ++ **Miscellaneous** + * **[XrdSecgsi]** Re-activate xrdgsitest + * **[RPM]** Include xrdgsitest in xrootd-client-devel package. + * **[XrdFileCache]** Add example of filecache configuration. + +------------- +Version 4.5.0 +------------- + ++ **New Features** + * **[XrdCms]** Allow specifying a different timeout for null cached entries; fixes #413 + * **[XProtocol/XrdSec/Server/XrdCl]** Implement request signing. + * **[XrdCl]** Add ZIP extracting capability to xrdcp. + * **[XrdCl]** Include the release number in client Login request cgi. + * **[XrdCl]** Add support for spaces in file names for mv operation. + ++ **Major bug fixes** + * **[XrdCrypto/Secgsi]** Fix XrdCryptosslMsgDigest::Init ; set 'sha256' as + default algorithm. + * **[XrdCl]** Use posix semaphores for fedora >= 22. Disable + omit-frame-ponter for gcc >= 4.9.3 if custom semaphores are used. + ++ **Minor bug fixes** + * **[XrdSecsss]** Fix memory leak in sss protocol. + * **[XrdNet]** Allow hostnames to begin with a digit. + * **[XrdCl]** Fix segfault in case a user cannot be mapped to a home directory. + * **[XrdCl]** Make sure a socket is always associated with a proper poller + object (not null). + * **[XrdCl]** Fix deadlock in XrdCl::PollerBuiltIn during finalize. + * **[XrdCrypto]** Do not use md5 checksum on OSX platform. + ++ **Miscellaneous** + * **[RPM]** Include xrdacctest in xrootd-server package. + * **[RPM]** Add conditional BuildRequires for ceph >= 11. + * **[RPM]** Use compat-openssl10-devel for fedora>=26. + * **[XrdCl]** Make sure the Log class can be used by any client plugin implementation. + +------------- +Version 4.4.0 +------------- + ++ **New Features** + * **[Server]** Add new [no]rpipa option to xrd.network directive. + * **[Server]** Allow objectid's to be specified in the authorization file. + * **[Server]** Add new logging plugin interface. + * **[Server]** Fixes #345 - add sid to TOD structure (ABI compliant). + * **[Server]** Implement resource selection affinity (primarily for ssi). + * **[XrdCl]** Add Metalink support (xrdcp & API). + * **[XrdCl]** Enable metalink processing on default. + * **[XrdCl]** xrdcp: use cks.type cgi tag to select the checksum type. + * **[XrdCl]** Support local metalink files. + * **[XrdCl]** Add support for GLFN redirector of last resort. + * **[XrdCeph]** Implemented pools of ceph objects. + ++ **Major bug fixes** + * **[Posix]** Remove double unlock of a mutex. + * **[Client]** Serialize security protocol manager to allow MT loads. + * **[Authentication/sss]** Fix dynamic id incompatibility introduced in 4.0. + * **[XtdHttp]** Removed the deprecated cipher SSLv3, in favor of TLS1.2 + * **[XrdCl]** Close file on open timeout. + * **[XrdCl]** Differentiate between a handshake and an xrootd request/response + while processing an incoming/outgoing message. + * **[XrdCl]** Fix dangling pointer issue that occurs while forking. + * **[XrdCl]** Ensure DefaultEnv is finalized after last use of the object. + ++ **Minor bug fixes** + * **[Proxy]** Avoid SEGV when printing memory cache statistics. + * **[Server]** Avoid XrdNetIF static initialization issues. + * **[Server]** Honor DFS setting when forwarding operations. + * **[Server]** Make sure lockfile time is updated in deprecated runmodeold. + * **[Server]** Fixes #344 - squash path before checking for static redirect. + * **[Server]** Free Entity before replacing it from the cache (memleak). + * **[XrdCl]** xrdfs ls does not include opaque info in a listing. + * **[XrdCl]** Eliminate unnecessary write notifications. + * **[XrdCl]** Forward xrd.* parameters from the original to the redirection URL. + * **[XrdCl]** Do not preset CWD in batch mode. + * **[XrdCl]** Be complaint with file URI scheme. + * **[XrdCl]** Fix wrong query string used for opaquefile code. + * **[XrdCl]** Translate XRootD error code to errno before passing to strerror. + * **[XrdCeph]** Fixed thread safety of filedescriptors in the ceph plugin. + * **[XrdCeph]** Protected initialization of ioCtx object and striper objects + by mutex in the ceph plugin. + * **[XrdCeph]** Fixed memory corruption in asynchronous read from ceph. + * **[XrdXml]** Make sure c-string buffes are properly terminated. + * **[XtdHttp]** Don't trim printable characters. + ++ **Miscellaneous** + * **[XrdCl]** Change the way handlers and messages are matched (use maps). + * **[Apps]** Add xrdacctest to the tools set to test access control databases. + * **[Packaging/RPM]** Set max open files limit to 65k for systemd services. + +------------- +Version 4.3.0 +------------- + ++ **New Features** + * Add option to query network configuration via configured interfaces. + * **[Proxy]** Default event loops to 3 and allow it to be set via config. + * **[Server]** Let client inform redirector why it's retrying a lookup + using the triedrc CGI element. + * **[Server]** Add cms.cidtag directive to qualify the global cluster id + (solves dpm problem). + * **[Server]** Make it possible to effeciently query an external database + for file existence via the statlib plug-in (largely for DPM).` + * **[Server]** Allow declaring extra large I/O buffers (mostly for Ceph). + * **[Server]** Allow iovec based data responses (no ABI changes). + * **[Misc]** Add back trace capability to the tool set. + * **[Misc]** Add generalized XML parsing ability. + * **[Misc]** Add metalink parsing for future integration. + * **[XrdCl]** xrdcp add env var to disable recovery + * **[XrdCl]** Add support for multiple event loops. + ++ **Major bug fixes** + * **[Server]** Correct IP address matching between IPv4 and IPv6. Fixes #300. + * **[Server]** Ruggedize cmsd thread synchronization during node deletion. + * **[Server]** Delete extraneous semaphore wait to avoid deadlock (#290). + * **[Server]** Return correct response to a delayed open. + * **[Server]** Fix build of kXR_wait message in case of delayed open. + * **[XrdCl]** Detect whether client is dual stacked based on outgoing + connection in addition to DNS registration. + * **[XrdCl]** Avoid EAGAIN loop. Fixes #303. + * **[XrdCl]** Append opaque info in case we retry at a data server after + being redirected. + * **[XrdCl]** Fix: FileStateHandler::OnOpen seqfaults when both write timeout + and OpenHandler timeout at the same time. + * **[XrdCl]** Avoid SEGV when server fails after it responds waitresp. + * **[XrdCl]** Continue processing remaining files after error occurrence. + * **[XrdCl]** Fix for dangling pointer problem in deep locate, fixes #324 + * **[Misc]** Add possibility to specify disk usage parameters in .. G, T units + using XrdOuca2x::a2sz(). + * **[Python]** Fix lock inversion in python bindings. + * **[Python]** Check if python interpreter is still initialized. + ++ **Minor bug fixes** + * **[All]** Fix numerous issues with space reporting (spaceinfo, query space, + statvfs) such a double counting, scaling, and format issues. + * **[Proxy]** Do not use the ffs code path if nothing is writable. This avoids + initialization failure when the origin is a large WAN cluster. + * **[Server]** Be agnostc NTP defaults when rotating logs (fixes new RH7 defaults). + * **[Server]** Pass correct total data length in iovec to Send(). + * **[Server]** Avoid redirection loop during error recovery in a uniform cluster. + * **[Server]** Make sure N2N gets configured for the cmsd when actually needed. + * **[Server]** Properly handle an inifit NPROC limit. Fixes #288. + * **[Server]** Make sure the cluster ID is always formatted the same way. + * **[Server]** Correctly compute timeout wait. + * **[Server/Logrotate]** Make sure rotating pattern is not expanded in an if statement, fixes #302 + * **[Misc]** Include XrdXmlReader in the spec file. + * **[Misc]** Fix bug in access statistics print. + * **[Misc]** Allow conversion of decimal numbers in XrdOuca2x::a2sz() + * **[XrdCl]** xrdfs prepare has to be provided with a filename, fixes #309 + * **[XrdCl]** Make sure recursive copy is disallowed only for checksum with user provided value, fixes #304 + * **[XrdCl]** Use the same timeout value for all close operations in xrdcp with TPC enabled. + * **[XrdCeph]** Fixed race condition in multistream access to files fo CEPH + ++ **Miscellaneous** + * **[Server]** Prevent cmsd reconnect storm when things get way slow. + * **[Server]** Changes to allow for Solaris compilation. + * **[Server]** Changes to allow for OSX compilation. + * **[Server]** Detect cyclic DNS host registration when processing '+' hosts. + * **[Server]** Display manager IP addresses during '+' host resolution. + * **[Util]** Avoid compiler warning about unsafe mktemp. + * **[App]** Do not report expected errors as errors. + * **[App]** Always show any unusual node status in the display. + * **[Client/Python]** Add MANIFEST.in for python bindings. + * **[XrdFileCache]** Implement blacklisting in a FileCache decision plugin. + * **[XrdFileCache]** Make sure requested offset is reasonable. + * **[XrdFileCache]** Return -1 and set errno when bad offset is passed in. + * **[XrdFileCache]** Only generate error for negative offsets, as per posix. + * **[XrdFileCache]** Add startup protection for ReadV, too. It was already there for Read. + * **[XrdFileCache]** Fix bug in cache scanning; simplify deletion loop. + * **[XrdFileCache]** Use bytes to calculate how many files to purge, not blocks; + subtract actual size of the file, not the length of it returned by stat. + * **[XrdFileCache]** In cache purge, use stat.mtime of cinfo file if last access time can not + be determined from contents of cinfo file. + * **[XrdFileCache]** Fix argument type from int to long long (was n_blocks, is size_in_bytes now). + * **[XrdCl/XrdSys]** Use custom semaphores only for glibc<2.21. + * **[XrdCl]** Remove libevent-based poller implementaion. + * **[XrdCl]** Report reason for reselection via triedrc CGI element. + * **[XrdClient]** Changes to allow for Fedora rawhide C++11 compilation. + * **[XrdCeph]** Fixed XrdCeph compilation for C++11 enabled compilers + * **[XrdCeph/CMake]** Fix for undefined symbols (link XrdUtils). + +------------- +Version 4.2.3 +------------- + ++ **Major bug fixes** + * **[Server]** Avoid SEGV if cmsd login fails very early. + * **[Server]** Avoid SEGV when an excessively long readv vector is presented. + * **[Server]** Rationalize non-specfic locate requests. + * **[XrdCl]** Process waitresp synchronously via Ignore return to avoid SEGV. + * **[XrdCl]** Avoid memory leak when a handler returns Ignore for a taken message. + * **[XrdCl]** Fix "tried" logic by forwarding the errNo + +------------- +Version 4.2.2 +------------- + ++ **Major bug fixes** + * **[Proxy]** Protect forwarding proxy server from slow connections. This should + fix most, if not all, SEGV's that the server encountered under heavy load. + * **[Server]** Fixes #248 Prevent infinite loop when shift arg is negative. + * **[Server]** Complain when passed I/O length is negative. + * **[Server]** Avoid execution stall during node logout when the thread limit + has been reached. + * **[Server]** Make sure to capture return code for stat() to prevent random + results. + * **[XrdCl]** Make sure to get filestate lock during timeout processing to + avoid MT intereference and possible random results. + * **[XrdClient]** Restore commented out abort() when an attemp is made to index a + vector outside of its current bounds (avoids random results). + * **[Server/Proxy]** Delay deleting a file object if the close was not successful. + This avoids deleting objects that may have pending activity resulting in an + eventual SEGV. This is a bypass fix to another problem. + ++ **Minor bug fixes** + * **[Server]** Fixes #234 Properly register all components in a mkpath request. + * Correctly handle copying into a non-existent directory when automatic + path creation is enabled. + * **[XrdCl]** xrdfs correctly handles quotations (fixes the problem with ALICE token) + ++ **Miscellaneous** + * Fixes #245 Provide compatibility when cmake version is > 3.0. + * Use atomics to manipulate unlocked variable pollNum. + * Bugfix: release lock when a file is closed before the prefetch thread is started. + Observed with xrdcp ran without -f option and an existing local file. Fixes #239. + * Protect from reads exceeding file size. Fixes #249. + * Release Stream lock before invoking callbacks. Fixes #216 + * TPC: Fix deadlock in case of error in the TPC authentication + * Increase max size of write to disk queues. + * Fix bug in endswith. Fixes #260 + * XrdCeph : fixed problem with files bigger than 2GB for synchronous writes + * **[XrdCl]** Change message loglevel from Error to Debug. Fixes #246. + * **[XrdCl]** Fix race condition in PostMaster initialization + * **[XrdCl]** Provide atomicity for PostMaster value using built-in functions + * **[XrdFileCache]** fixed deadlock on immediate file close (e.g. xrdcp to non-writable output) + * **[XrdFileCache]** fixed errors on some posix operations using virtual mount + +------------- +Version 4.2.1 +------------- + ++ **Miscellaneous** + * **[Client/Cl]** Make sure kXR_mkpath is set for classic copy jobs when the + destination is xrootd (backward compatibility fix). + +------------- +Version 4.2.0 +------------- + ++ **New Features** + * **[Client/Python]** Integrate xrootd-python into the main package. + * **[Server]** Include a Ceph OSS plug-ing. + * **[Server]** Implement throttling. + * **[Server]** Detect redirect loops using "tried" token. + * **[Server]** Implement the "cid" option for config query to display the + unique cluster ID. + * **[Server]** Allow suspending and enabling remote debugging without a + restart. + * **[Server]** Implement black/whitelist with optional redirection. + * **[Server/Proxy]** Add the xrdpfc_print tool to print the caching + proxy metadata. + * **[Server/PlugIns]** Provide a mechanism to pass command line arguments + to plug-ins. + * **[Server/PlugIns]** Provide access to the native and the active extended + attribute implementation. ++ **Major bug fixes** + * **[All]** Fix various memory access issues. + * **[Server]** Fix various IPv4/IPv6 compatibility issues. + * **[Server]** Avoid disabling of frm notifications due to plug-in + initialization issues. + * **[Server/Proxy]** Avoid holding a global lock when opening/closing files + to solve timeout issues. + * **[Security/GSI]** Fix reloading of CA and CRLs. ++ **Minor bug fixrs** + * **[Server/HTTP]** Fix issues related to invalid chunk sizes. + * **[Server/Proxy]** Various logic and permission processing fixes. ++ **Miscellaneous** + * **[Client/Cl]** Make the compiler issue warnings when the return codes + from the File and FileSystem methods are unchecked. (issue #188) + * **[RPM]** Disable building of the compat package by default. + * **[Server/Proxy]** Avoid serializing stat() via the proxy to improve + performance. + * **[Tests]** Factor out the common testing code from the client tests so + that it can be re-used. + +------------- +Version 4.1.2 +------------- + ++ **Major bug fixes** + * **[Utils]** Don't confuse -I and --tpc while parsing commandline parameters + for xrdcp. (issue #213) + * **[Server]** Fix various IPv4/IPv6 issues. (issues #164, #227) ++ **Minor bug fixes** + * **[Client/Cl]** Print mtime when doing xrdfs stat. + * **[All]** Fix some memory access issues. (issues #186, #197, #205) + * **[Server]** Recreate logfile fifo if it already exists and is a file. + (issue #183) + * **[Server]** Properly reset suspend state when reconnecting cmsd. + (issue #218) + * **[Server]** Avoid disabling async I/O when using an oss plugin that does + not implement file compression. (issue #219) + * **[Server]** Do not debit space when relocating a file within the same + partition. + * **[Server]** Fix meta-manager port directive ordering. + * **[Server/Logrotate]** Do not print anything to stdout to avoid making cron + send emails to admins. (issue #221) ++ **Miscellaneous** + * **[Server/Proxy]** Disable POSC processing when a proxy plugin is loaded. + +------------- +Version 4.1.1 +------------- + ++ **Major bug fixes** + * **[RPM]** Remove the library patch from xrootd-config to enable multiarch + installations. + * **[RPM]** Move the user creation scriptlets to xrootd-server where they + belong. (issue #179) + * **[Server]** Fix PowerPC compilation. (issue #177) + * **[Server]** Avoid the pitfalls of infinite nproc hard limit in Linux. + * **[Server]** Correct flag definition to include cms plugin loading. (issue #176) ++ **Miscellaneous** + * **[Man]** Update documentation. + * **[Client/Cl]** Set the multi-protocol ability basing on an environment + variable. + +------------- +Version 4.1.0 +------------- + ++ **New Features** + * **[Everyting]** Implement dynamic plugin shared library filename versioning + to allow multiple major versions to co-exist. + * **[Server]** Compelete IPv6/IPv6 and public/private network routing. + * **[Server]** Allow the checksum manager to use OSS layer to access data. + (issue #140) + * **[Server]** Allow the definition of subordinate clusters. + * **[Server]** Support multiple checksum types. Client can select non-default + checksum using the "cks.type=" cgi element. + * **[Server]** Provide plugin interface for handling extended attributes. + * **[Server]** Add options to xrd.network to control keepalive. + * **[Server]** Control core file generation via xrd.sched core directive. + * **[Server]** Add pss.permit directive to restrict outbound connections for + forwarding proxies. + * **[Server]** Allow xrootd to handle objectid names as exports. + * **[Server]** Install and package the cluster mapping utility: xrdmapc. + * **[Server]** Allow the specification of xrootd.seclib default. + * **[Server]** Pass along XRD_MONINFO setting and application name to + monitoring. + * **[Server/Proxy]** Implement a forwarding proxy option. + * **[Server/Proxy]** New configuration of XrdFileCache using 'pfc.' prefix. + * **[Sever/HTTP]** Support gridmap parsing. + * **[Client/Cl]** Inform the server about availability of local IP address + types (IPv6/IPv4, public/private) to in order to facilitate redirections. + * **[Client/Cl]** Make the client send kXR_endsess request when recovering + broken connection - avoids 'file already open' errors. + * **[Client/Cl]** Implement TCP keep-alive support. + * **[Client/Cl/xrdcp]** Optimize xrdcp uploads by compensating for latency. + * **[Client/Cl/xrdcp]** Make it possible for xrdcp to run multiple transfers + in parallel using the '--parallel' option. + * **[Client/Cl/xrdcp]** Make it possible for xrdcp to concatenate multiple + sources to stdout. + * **[Client/Cl/xrdfs]** Add xrdfs locate -i option to ignore network + dependencies (IPv6/IPv4). + * **[Security]** Add new security framework loader to allow external pacakges + that linked against security plugins to dynamically load them instead. + * **[Security/sss]** Allow forwardable sss tokens when ecrypted with a + forwarding key as defined by the xrdsssadmin command. + * **[Plugins]** Implement generic matching rules to version check 3rd party + plug-ins. + * **[Packaging/RPM]** Add SystemD configuration files for RHEL7. + * **[Packaging/RPM]** Introduce compat RPM packaging providing xrootd 3.3.6 + deamons and libraries with the ability to switch between desired versions + using the sysconfig file. + * **[Packaging/RPM]** The RPM naming has been switched back to xrootd + (from xrootd4). + * **[Utils]** Add xrootd-config utility. + ++ **Major bug fixes** + * **[Server/HTTP]** Make it possible to handle files larger than 2GB. + * **[Server]** Prevent blacklisting of all connctions when role is supervisor. + * **[Server]** Fix bug in handling cms.dfs redirect verify that would keep + the client is an infinite wait loop. This also affected locate requests + regardless of what the redirect option was set to. + * **[Server/Proxy]** Avoid SEGV when no environment has been passed in the + proxy server. + ++ **Minor bug fixes** + * **[C++ API]** Provide complete portability and correct behaviour across + platforms with and without Atomics. This patch does not change any ABI's. + * **[Server]** Do not set *TCP_NODELAY* for unix domain sockets as this + issues a nasty error message. + * **[Server]** Allow cms.dfs mdhold argument to be 0 as documented. + * **[Server/Plugins]** Add missing initializer to the LocInfo structure. + * **[Server/Plugins]** Correct header define gaurd in XrdSfsFlags.hh. + * **[Server/Proxy]** Fully support extended file system features and pass + those features through a proxy server. (issue #115) + * **[Client/Cl]** Remove duplicates from the HostList. + * **[Client/Cl]** Fix minor atomicity issues (C++11). + ++ **Miscellaneous** + * **[Server]** Actually remove xmi plugin handling as xmilib is no longer + supported. + * **[Server]** Make sure to always passhrough CGI information. + * **[Server]** Honor network routing when creating the client's i/f + selection mask. + * **[Server]** Efficiently handle replicated subscribers (i.e. managers). + * **[Server/HTTP]** Remove useless loading the security framework. + * **[Server/Security]** Add new NetSecurity::Authorize() method that accepts + text. + * **[Server/Proxy]** Properly support proxying objectids. + * **[Server/Proxy]** Clean-ups in the caching proxy. + +------------- +Version 4.0.4 +------------- + +* **Major bug fixes** + * **[Client/Cl]** Properly allocate buffers for error messages. (issue #136) + * **[Client/Cl]** Check if there is enough data before unmarshalling. + * **[Client/Cl]** Fix a memory leak in MessageUtils::WaitForResponse + affecting all synchronous calls. + * **[Client/Cl]** Prevent a segfault in the destructor when called after + the libXrdCl library has been finalized by the linker - ROOT garbage + collection. https://github.com/cms-externals/xrootd/pull/1 + * **[Client/Posix]** Fix broken readdir_r() and readdir_r64() functions. + * **[Server]** Use correct flag when adding a cluster. The bug made it + impossible to have more than one supervisor node. + * **[Server/Logrotate]** Prevent stack corruption by correctly sizing the + timestamp buffer. + ++ **Minor bug fixes** + * **[Client/Cl]** Properly check if a recursive copy was requested to avoid + unnecessarily stating the source. + * **[Client/Cl]** Avoid inserting duplicate entries to HostList when retrying + at the same server. + * **[Client/Cl]** Normalize (trim leading zeroes) before comparing adler and + crc checksums. (issue #139) + * **[Client/Posix]** Prevent mkdir failure in a clustered environment by + creating the full directory path by default. + * **[Client/Possix]** Fix a memory leak when doing deep locate. + * **[Server/Logrotate]** Use expect to send a ping to pipes. This prevents + logrotate from hanging when nobody is listening at the other end of the + pipe. + * **[Authentication/Client]** Pass the external environment to the protocol + manager. (issue #133) + * **[Authentication/sss]** Fix a memory leak. + * **[Utils]** Avoid SEGV when assigning a unix domain address to a + NetAddrInfo object previously used to hold a TCP domain address. + * **[Server/cmsd]** Use the same write selection rules for dfs and non-dfs + environments. + ++ **Miscellaneous** + * **[Server/Logrotate]** Prevent the default configuration from sending + emails to admins and from creating a new log after the old one has + been rotated. (issue #135) + * **[Server/SELinux]** Using expect in logrotate requires the logrotate_t + context to have access to pseudoterminals and tmpfs as well as stating + fifos + * **[Client/Commandline Parser]** Allow local to local copy in new xrdcp but + not in the old one. + * **[Client/Cl]** Discard a whole cluster on failure in federation context. + (issue #132) + +------------- +Version 4.0.3 +------------- + ++ **Major bug fixes** + * **[Server]** Make sure the network routing is honored in all cases. This + fixes problems encountered by sites whose clients use a private IP address + to connect to a redirector's public IP address. (issue #130) + +------------- +Version 4.0.2 +------------- + ++ **Minor bug fixes** + * **[Client/Cl]** Handle all non-NULL-terminated error responses correctly. + * **[Client/Cl]** Release old auth buffer when reconnecting after TTL + expiration. + ++ **Miscellaneous** + * **[Client/Cl]** Retry after an incomplete local write. This produces + clearer error messages. Ie: "Run: [ERROR] OS Error: No space left on + device" instead of: "Run: [ERROR] OS Error: Operation now in progress". + * **[Client/Cl]** Don't force a server to issue a short read when fetching + last data chunk. This works around issues for proxied FAX sites. + +------------- +Version 4.0.1 +------------- + ++ **Major bug fixes** + * **[Server]** Prohibit accessing memory via /proc using digFS. + ++ **Minor bug fixes** + * **[Server]** Prevent over-scan of the xrd.network routes option which may cause + a config file error message and initialization failure. + * **[Server]** Fixes to make things compile on ix86, arm and ppc64. + * **[Server]** Correct protocol name supplied to monitoring for userid. + * **[Server/Proxy]** Various minor fixes to caching proxy. + * **[Security]** Check the length before looking inside a SUT buffer. (issue #126) + * **[Client/Cl]** Check for copy source and target validity to display proper error + messages. + * **[Client/Cl]** Return default plug-in factory for an empty URL. (issue #120) + * **[Client/Posix]** Provide full error mapping for POSIX interface. + * **[All]** Remove some unnecessary commas and semicolons. (issue #121) + ++ **Miscellaneous** + * **[Server]** Pass client login information to monitoring. + * **[Client/Cl]** Make xrdfs locate -h synonymous to locate -m. + * **[Client/Cl]** Add -i option to xrdfs locate setting the Force flag. + * **[Docs]** Various documentation updates. + +------------- +Version 4.0.0 +------------- + ++ **New Features** + * Supprt IPv6. Please read docs/README_IPV4_To_IPV6 for details. + * Introduce the XrdFileCache library - a proxy server plugin used for caching + of data into local files. + * Beta support HTTP(S). + * Provide protocol bridge to let other protocols use xrootd back-end plugins. + * Provide full support for public/private IP networks. + * Allow remote debugging via the xrootd.diglib directive. + * Provide a mechanism to manually control log file rotation via -k and add + support for logrotate. + * Add -z option to enable high recision log file timestamps. + * Define a new plug-in to allow replacement of the stat() function when + used to determine exported file characteristics. This plug-in is meant + to be used by tape-backed file systems that identify offline files in + odd ways (e.g. GPFS). Patch assumes XRDROLE patch below. + * Implement full readv-passthru for enhanced performance. + * Add a disconnect record to the f-stream. + * xrdcp is now the same as xrdcopy, and old xrdcp is now xrdcp-old + * Make clients configurable via /etc/xrootd/client.conf and + ~/.xrootd/client.conf + * Implement a plug-in system for client's File and FileSystem queries. + * Make it possible for 'xrdfs stat' to query for combination of flags. + * Make third party copies cancellable. + * Implement xrdfs spaceinfo, cat and tail commands + * Terminate iddle connections after a timeout and treat timeouts on streams + that should be active (because of outstanding requests with no delay times) + as errors. + * Implement XrdCl::File::Visa and XrdCl::File::Fcntl. + * Support for full URL redirects. + * File and Filesystem objects implement property system to pass custom + information to and from them (including plug-ins) without breaking + ABI. + * Add --dynamic-src to xrdcp options to allow dynamic file copying. + * Implement the directory listing in bulk. + * Enable locate to return host names not just IP addreses. + * Implement node blacklisting for the cmsd (see cms.blacklist directive). + * Add mv command to frm_admin. + * Allow query of current role and dynamic cms state via kXR_query. + * Implement query config chksum to return supported chksum name. + * Add version as a variable that can be returned by kXR_Qconfig. + * Add sitename as an argument to kXR_Query+kXR_Qconfig. + * Provide disconnect notifiation to underlying file system. + * Provide the filesystem plugin a way of creating a session storage area. + * Add flag to indicates a secondary copy of a file exists. + * Allow testing for undefined set/env vars via if-else-fi. + * Add '-L' flag to the xrootd command to allow loading a protocol library + * Add flag to indicates a secondary copy of a file exists + + ++ **Bug fixes** + * Fix various dead locks in the IOEvents poller. + * Implement LinuxSemaphore class in order to replace buggy POSIX semaphores + on Linux. + * Honor the cmsd.dfs directive for locate request to avoid placing a + file in ENOENT status. + * Make sure that the old client runs only in IPv4 mode as mixing modes does + not work for a variety of reasons. + * Accept old-style as well as new-style IPv6 addresses in the sss + protocol. This allows the new client to use this protocol after + it implemented IPv6 support. + * Prevent invalid mutex operations in auto-termination routine. + * Resolve naming conflicts within the frm that resulted from the + statlib plugin implementation. + * Do not rely in file locking to serialize inter-thread access. This + fixes the prolem of usage file drift. + * Fix various parse context issues in copy config with --recursive. + * Recognize object deletion in the error handling path. + * Use atomic FD_CLOEXEC where available to prevent FD leaks. + * Squelch casting complaints from C++11. + * Make sure to return all nodes in a star locate request. + * Always load protocols in the specified order. + * Fix xrootdfs wcache crashing issue when using virtual file descriptor. + * Fix selection of a server when a DNS entry resolves to more than one. + * Correct pthread_cond_timedwait() time calculation and error handling. + * Fix null insertion of hostname in error message when open fails. + * Fix issues with extensions in GSI proxies + * Fix problem with creation of the forwarded KRB5 ticket + * Correctly handle reading of a partial readv headers (issue #45) + * Make sure to propagate username and password when redirecting + * Honor request timeouts when processing kXR_wait + ++ **Miscellaneous** + * XrdClient and associated commandline utilities are now obsoleted. + * Propagate info about partial success from deeplocate to dirlist. + * Remove perl interface. + * Send timezone, country code and application name while logging in. + * Change interfaces to copy process to use property system (allows for + adding features without breaking the ABI). + * Final change to f-stream monitoring. Replace standard deviation + (sdv) calc with reporting sum of squares (ssq) counts. + * Make public headers compile cleanly with -Wall -Wextra -Werror. + * Support passing cert, key paths via URLs + * Allow testing of undefined set/env vars via if-else-fi + * Pass user environment settings settings in the login CGI + * Use DNS names instead of addresses for kXR_locate when listing + +------------- +Version 3.3.6 +------------- + ++ **Minor bug fixes** + * Prevent SEGV when error occurs during stat (issue #70) + * Prevent SEGV in redirect monitoring (issue #61) + * Set reasonable linux thread limit and warn it we cannot do so. + ++ **Miscellaneous** + * Support for C++11 (narrowing fixes, unique_ptr vs. auto_ptr) + * Support for CMake 2.8.12 (interface link libraries) + +------------- +Version 3.3.5 +------------- + ++ **Minor bug fixes** + * Fix minor Coverity issues in XrdCl + * Fix a rarely occuring segfault when forking XrdCl under heavy load + * Fix various issues related to group name retrieval (issues #51, #52, #53) + ++ **Miscellaneous** + * Make XrdSys/XrdSysIOEvents.hh private - could not have been used anyways + * Add a sysconfig template to preload custom allocators in order to fix + memory issues on RHEL6 + * Allow up to 63 characters for a site name + +------------- +Version 3.3.4 +------------- + ++ **Major bug fixes** + * Serialize sss authentication client initialization to prevent race + conditions + * Actually cancel the JobManager threads while stopping it - this affected + client side fork handling (new client) + * Restore original meaning of -adler and -md5 to xrdcp (issue #44) + ++ **Minor bug fixes** + * Append CGI info when retrying at a server that handshaked but never + respnded to the request (xrdcp) + * Do socket accepts asynchronously to prevent DNS resolution from blocking + accepts (issue #33) + * Warn about incomplete dirlist responses (xrdfs) + * Cast the utilization statistics to uint16_t before printing to + print actual numbers instead of letters corresponding to ASCII codes + (xrdfs) + ++ **Miscellaneous** + * When calling File::Stat use file handle instead of path + * Improve handling of malformed kXR_readv responses (new client) + * Explain parameters of xrdcopy --tpc (documentation, issue #46) + +------------- +Version 3.3.3 +------------- + ++ **Major bug fixes** + * Prevent SEGV's when reusing a recycled protocol object under certain + conditions (xrootd server) + * Prevent SEGV when using the -DS/-DI commandline parameters in xrdcp + (issue #13) + * Prevent integer overflow when calculating client recovery windows + * Make sure the new client tries all available authentication protocols + when connecting to a security enabled server (issue #14) + * Detect buffer size mis-matches when server returned valid response with + invalid size (xrdcopy) + * Recognize /dev/null and /dev/zero as special files when using copy + commands + ++ **Minor bug fixes** + * Prevent the new client deadlock on Solaris and MacOS when using + the built-in poller and connecting to localhost (issue #5) + * Compensate for ROOT garbage colletion issues when calling the + new client code + * Avoid favoring socket writes when using new client with the built-in + poller + * Strip off opaque information from dest filename when copying to local + filesystem using xrdcp (issue #21) + * Fix setting client timeout resolution while connecting to a server + ++ **Miscellaneous** + * Change the RPM package layout to match the one used by EPEL (issue #12) + * Drop the daemon user RPMs + * Allow new client connection parameters to be tweaked by connection URL CGI + * Make the built-in poller default again in the new client - after resolving + issue #5 + +------------- +Version 3.3.2 +------------- ++ **Major bug fixes** + * Fix the opaque information setting in xrdcp using -OD (issue #1) + * Fix compilation on Solaris 11 (issue #7) + * Fix issues with semaphore locking during thread cancellation on + MaxOSX (issue #10) + * Solve locking problems in the built-in poller (issue #4) + * Solve performance issues in the new client. Note: this actually + changes some low level public interfaces, so the soname of + libXrdCl.so has been bumped to libXrdCl.so.1. The xrootd.org + RPMs also provide the old libXrdCl.so.0 in order to preserve the + binary compatibility with the clients linked against it. + +------------- +Version 3.3.1 +------------- ++ **Major bug fixes** + * Correct XrdClient ABI incompatibility issue introduced in 3.3.0 + * Install additional private headers + +------------- +Version 3.3.0 +------------- ++ **New Features** + * Stable interfaces immutable in minor releases (except XrdCl). Only + public header files are installed in the usual include directory. + In order to ease up transition of some clients some of the private + include files are also installed in private subdirectory. + * New asynchronous and thread-safe client libraries and executables + (XrdCl). The ABI compatibility is not guaranteed until 4.0.0. + * Build the xrootd protocol plugin as a shared library. + * Add the altds directive to allow pairing a cmsd with an alternate data + server. + * Differentiate between packed and unpacked readv monitoring records. + * Allow plugin libraries to be preloaded. This feature is only meant + for MacOS. + * Include optional site name in summary monitoring records. + * Include optional site name in server identification record if the + site name was specified on the command line (-S) or via config + file (all.sitename directive). + * Define a standard supported mechanism to obtain the default storage + system object. + * Provide an ABI-compatible interface to obtain a default cmsd client + object. This patch does not change the definition of the XrdCmsClient + object and is ABI compatible with all previous releases (DPM support). + * Allow multiple comma separated protocols in XrdSecPROTOCOL client-side + envar. This allows the client to select 1 of n protocols. + * Implement new "f" stream monitoring. + * Add new summary counters for readv and readv segs. + * Add boiler plate comments indicating the all software is licensed under + LGPL. No functional source code was modified by this patch. + * Add GPL and LGPL license text. + * Liberlize locking structure to prevent lock inversion relative to + external locks. + * Provide libevent replacement for Linux (epoll), Solaris (poll_create), + and others (poll). Note: versions of Solaris less than 10 are no longer + supported and they will no longer compile with this update! + * Provide a libevent type replacement package. + * Allow tracker files (e.g. ".fail") to be placed in a shadow directory. + This is controlled by the new fdir option on the oss.xfr directive. + * Allow meta-files (i.e. .fail file) to be relocated to a shadow directory + using the oss.xfr directive. This avoids polluting the exported name + space when an frm transfer operation fails. + * Create a general place for platform dependent utility methods. + * Add third party copy statistics to the summary record. + * zlib compatible checksum plugin + ++ **Major bug fixes** + * Serialize access to cache entries to prevent SEGV's. + * Fix the fast response queue so that it doesn't run out of response + slots causing a big performance penalty. This is a high priority fix. + * Properly disarm the mutex helper when the mustex object is deleted. + * Use correct variable to hold osslib parameters. This patch fixes commit + 2e27f87a (version checking) and without this patch makes it impossible + to load an oss plug-in. + * Properly check for errors when client read returns 0 and reflect true + status. This only affects the Posix client interface. + * Remove redundant flag indicating a running poller. This may cause the + poller to never be woken up when a timeout value changes. + * Fix tag in ofs statistics. It is improperly terminated and may + cause certain xml parsers to fail; rendering monitoring useless. + * Undo the side-effect of commit ff8bdbd6 that prevented the frm from + sending stage notifications to xrootd; causing opens and xrdstagetool + to hang with dynamic staging enabled. + * Make sure the id buffer is large enough to hold all id combinations. + * Avoid deadlock when closing a Posix File with an active preread. + * For concurrent queries for the same file allow servers to respond to the + query and only redirect clients to a stageable server if the file is not found. + ++ **Minor bug fixes** + * Add EPOLLRDHUP to avoid leaving sockets in CLOSE_WAIT with a one-shot + poll framework. + * Fully integrate checksum processing into a manager node. When configured, + it does not matter whether a client directs a checksum request to a manager + or a server. This also fixes bug report #93388. + * Make sure to reflect proper range of errors during read/write operations. + This also provides filesystem plugins full range of allowed return codes. + * Initialize the rMon toggle to avoid valgrind complaint. + * Fix minor issues reported by Coverity. + * Make sure opendir() returns a null pointer when the directory doesn't + exist. + * Make sure that XrootdFS returns ENOENT when opendir() returns a null. + * Make sure to use correct time to set mtime/atime after a physical reloc. + * Prevent hangs when doing exterme copy from server to server. + * Fix the -force option to really work for the mark subcommand. + * Pass through error code returned by the N2N plug-in. This only affects + the proxy server and caused feature interference. + * Automatically exclude originating server/cluster on an enoent static + redirect. + * Correct typos XRDPSOIX envars should really be named XRDPOSIX. + ++ **Miscellaneous** + * Remove superfluous includes or other move includes to eliminate + unnecessary dependencies in ".hh" files. This patch is required + to create an EPEL conformable include directory. + * Add port to prepare request struct as documented in 2.9.9. + * Add pathid to readv request struct as documented in 2.9.9. + +------------- +Version 3.2.6 +------------- ++ **Major bug fixes** + * GSI authentication: fix possible race condition while re-loading CA + certificates; fix also related memory leaks. + * GSI authentication: make sure the CA cache is not initialized twice (e.g. + server and client inside there), and that the cache entry pointers are + always initialized. + * Crypto OpenSSL modules: use more appropriate way to read the RSA complete key, + solving various issues for RH6 and derivations, included SL(C)6. + * Make sure redirect opaque information is passed along for all filename + based requests. This is required for DPM and EOS N2N services to work + in all cases (most importantly, stat). + * Make sure buffer ends with null byte before read suspension. This only + occurs on very heavily loaded connections. + * Fix the fast response queue so that it doesn't run out of response + slots causing a big performance penalty. This is a high priority fix. + ++ **Minor bug fixes** + * Properly detect external process failure and report correct error status + to a client. This also fixes bug report #91141. + * [XRootDPosix] Make sure to use a supplied cache even when no cache + directives given. + * Make sure to return a usable path string via XrdOucCacheIO::Path(). + * Actually support 4 different redirect destinations. + ++ **Miscellaneous** + * Transparent support for new name hashing algorithm adopted in openssl + 1.0.0x (GSI authentication protocol) + * Verbosity levels revised for GSI and PWD authentication protocols. + * Notification of initialization option for GSI and PWD authentication + protocols. + * Do not repudiate file existence on an "cancelled" error during open. + this patch addresses overloaded dCache pool nodes. + +------------- +Version 3.2.5 +------------- ++ **Major bug fixes** + * Make realoading gridmapfile atomic (protect from segfault) + * Propagate to clients proper range of errors during read/write operations + * Fix segfault when handling writes to files that have not been opened + +------------- +Version 3.2.4 +------------- ++ **Major bug fixes** + * Work around a dead-lock in the client fork handlers. + +------------- +Version 3.2.3 +------------- ++ **Major bug fixes** + * Make sure read statistics are updated for sendfile() and mmap I/O. + * Make sure refresh thread is dead before deleting deleting the keytab to + avoid SEGV's. + * Add missing include for compiling with gcc-4.7 (from Sebastien Binet). + This patch is required for successful compilation. + * Avoid segfaults when limiting number of redirections caused by failed + authorization. + * Avoid deadlock in the client fork handlers. + ++ **Minor bug fixes** + * Correct monitor initialization test to start monitor under all configs. + * Fix a memory leak in the client handshake algorithm. + ++ **Miscellaneous** + * Make RHEL6-created SRPMs buildable on RHEL5 by forcing RPM to use MD5 + digests. + * Fuse: Use default TTL values for data server connection and load + balance server connection. + +------------- +Version 3.2.2 +------------- ++ **Major bug fixes** + * Correct test whether or not to initialize redirect monitoring. The old + code never initialized it this disabling redirect monitoring. + * Backport frm notification fix that stalled stage-in requests from commit + 69e38cfd6b8bb024dd34f8eb28a666fbf97f346b + * Prevent SEGV when xrd.monitor rbuff value not specified + * Prevent xrdcp hangs when doing exterme copy from server to server. + * In case of 'limited proxy' look for VOMS attributes also in the parent + proxy. + * Correct log processing for sites that use the root directory as the + stomping ground for newly created files. + +------------- +Version 3.2.1 +------------- ++ **Major bug fixes** + * Don't build sendfile support on MacOSX because it doesn't work + * Prevent double-free abort when more than 16 files have been opened by a + client and the client terminates the session without closing the 17th one. + +------------- +Version 3.2.0 +------------- ++ **New Features** + * Retool the XrdOucCache object so that cache implementations can be + implemented as plugins. + * Add FSize method to the XrdOucCacheIO object to ease implementation + of disk caches containing partial files. + * Add the pss.cachelib directive to specify a cache plugin. + * Implement ultralow overhead redirect monitoring. + WARNING: ofs plugin writers will need to recompile their plugin interface + to be fully compatible with this commit due to additional + information passed to the ofs object "new" methods. + * Allow the XrdCmsClient interface (a.k.a Finder) to be a plug-in. + * Add ofs.cmslib directive to specify the XrdCmsClient plug-in. + * Add new class, XrdOucCallBack, to simplify using callbacks in the + XrdCmsClient plug-in. + * Define the frm.all.monitor directive to enable migration, purging, and + staging monitoring. This was originally part of xrootd.monitor but that + just was odd. Note that the stage, purge, migr events are no longer + accepted on the xrootd.monitor directive. + * Collapse he staging (s) and migration (m) records into a single transfer + (x) record. While not compatible, the previous implementation was new + code and no one actually was capturing these records. + * Implement a server identification record (=) that unquely identifies each + server. The record can be sent periodically and can be used as a heartbeat. + * Add -y option to xrdcp to limit number of extreme copy sources. + * Uniformly pass the execution environment to all oss and cms client + methods. This is largely for DPM support. + WARNING: While this update is binary backwad compatible to existing oss + plug-ins it is not source compatible. Plug-in writers will need + to modify their oss methods to successfully compile. + * Allow an automatic redirect when a file operation ends with ENOENT. + Allow redirects for chsum and trunc operations. + Both of the above are controlled via the xrootd.redirect directive. + * Report the timezone when connecting to a [meta]manager. + * Allow configuration of staging, migration, and purging events. + * Allow transfer script to inject information into the monitoring stream. + * Report number of attempted login, authentication failures, successful + authenticated and unauthenticated logins in the summary statistics. + * Indicate whether a disconnect was forced and whether it was a parallel + path (as opposed to a control path) in the monitoring record. + ++ **Major bug fixes** + * Provide compatibility for sprintf() implementations that check output + buffer length. This currently only affects gentoo and Ubuntu Linux. + We place it in the "major" section as it causes run-time errors there. + * Reinsert buffer size calculation that was mistakenly deleted. + This eventually causes a SEGV when detailed monitoring is enabled. + * Remove improper initialization that may cause a SEGV in the checksum + manager. + * Add missing initializer without which we will get a SEGV. This is a fix + for the just added monitoring code. + * Remove regressions that prevent a proxy cluster from being fully + configured. + ++ **Minor bug fixes** + * Correct debug message frequency that caused people to think some file + system partitions were being ignored. + * Correct pthread Num() to return thread-specific numbers. + * Make sure the sendfile interrupt counter is initialized to zero. + * Make sure to honor absolute cms.space values when percentage not + specified. + * Prevent double user map record when monitoring when auth is configured + but not actually monitored. + * Take timezone changes into account when waiting for midnight. This solves + the log rolling problem when changing between DST and standard time. + * Make sure to cut close records for open files during a forced disconnect + when monitoring file information. + * Do not create meta-files or update extended attributes when placing a + file into read-only space. + ++ **Miscellaneous** + * Bonjour code dropped + * Complete implementation of the fstat() version of stat(). + * Consistently pass the enviroment to the cms client enterface. + * Make return codes consistent between synchronous & async XrdCmsClient + returns. + * Document the XrdCmsClient interface in the header file. + * Cut close monitor records before cutting the disconnect record. + * Make frm_purged and frm_xfrd use sparate log files. + +------------- +Version 3.1.1 +------------- + ++ **New Features** + * Compile on Solaris 11 + * Add support for sending DN with monitoring information + * Add possibility to switch off automatic download of CRL from the web; + default is OFF; to enable it multiply by 10 the relevant CRL options + (i.e. 12 and 13 are like 2 and 3 but trying download if the file is not + found). + * Add refresh frequency time for CRL's; default 1 day . + ++ **Major bug fixes** + * Fix various client threading issues. + * [bug #87880] Properly unpack the incoming vector read data. + * Rework the handshake when making a parallel connection. Previous method + caused a deadlock when parallel connections were requested (e.g. xrdcp). + * Add HAVE_SENDFILE definition to cmake config. All post-cmake version of + xrootd until now have disabled use of sendfile() with resulting poor + performance. This fix corrects this. + * Don't force libXrdPss.so to be loaded for proxy managers. + * Fix various CMake issues: disable library inheritance, fix underlinking + problems, make sure libcom_err is present when building kerberos. + * Replace non-reentrant versions of getpwxxx and getgrxxx with reentrant + versions. This should prevent spurious uid/gid translations. + * Fix RedHat bug #673069: Missing header files required by DPM + * Don't ignore errors returned by kXR_close + * Init scripts: don't change the ownership of the sysconfig files + preventing the xrootd user from executing arbitrary code as root + ++ **Minor bug fixes** + * Add 'k' to the option list. It was wrongly deleted in the last option + refalgamization. + * Fix a typo in the specfile causing problems with multithreaded + compilation. + * Initialize xattr variable name so that xrdadler32 can fetch previous + checksum. The error caused xrdadler32 to always recompute the checksum. + * Make sure that monitor write length is really negative. + * Add the oss.asize hint to the destination URL in all possible cases. + * Properly print adler32 checksum in xrdcp. + * When the server certificate is expired, try to renew from the same path + before failing. + * Get the signing certificate for the CRL from its issuer hash, which can be + different from the CA hash. + * Add check for the format of the downloaded CRLs: DER or PEM + * Solaris init script: switch to xrootd user when invoked as root + * RHEL init scripts: always create /var/run/xrootd to handle /var/run + being mounted as tmpfs + ++ **Miscellaneous** + * Relax requirements on the permission mode of the x509 key files + * Disable client redirections reports to the console. + * Stop doing XrdFfsPosix_statall() if task queue is long. + * Get rid of compiler warnings + * Improve some log messages + * At server startup, only initialize the CA (and CRL, if required) for the + authority issuing the server certificate; additional CA's are initialized + only if needed. + +------------- +Version 3.1.0 +------------- + ++ **New Features** + * Use CMake to build the source code and retire all the other build systems. + * Add IOV as a selectable detail to xrootd.monitor directive. + * Provide a mode in xrootdfs to auto-update internal list of data servers. + and extend client connection TTL from one hour to infinity. + * Provide virtual xattr ("xroot.cksum") to obtain checksum for consistency. + * Make xrdadler32 use the new checksum format if it is set (fallback to old + format otherwise). In all cases, the old format is converted to the new + format whenever possible. + * Enforce r/o exports in the proxy server (finally added). + * Allow auto-fluching of I/O stream monitoring (default is off). + Patch submitted by Matevz Tadel, UCSD. + * Make proxy honor the export list at the storage layer. This allows sites + to disable staging via the proxy by specifying nostage for otherwise locally + stageable paths. + * Do not export the stage attribute to the meta-manager unless the path is + tagged with the stage+ attrbute on the export directive. + * WARNING: This update makes the oss plug-in source incompatible because an + additional parameter was added to the Stat() method. The update is binary + compatible and so only affects sites that recompile their plug-in. + * Allow the query checksum request to be issued via a proxy server. + * Add a query checksum interface to the POSIX interface. + * Defines the livXrdSecgsiAuthzVO plug-in to allow easy mapping from voms + vo names to users and groups. The plugin is configurable at run-time. + * Allow the OucErrInfo object to point to an environment. + * Add method to SysDNS to format an AF_INETx address into the RFC IPV6 + recommended format. + * Allow pointers to be placed in the OucEnv environment table. + * Extend the kXR_protocol request to allow the server to return detailed + information about node's role. This is backwardly compatible. + * The client uses kXR_protocol request to query for the server's role + (to distinguish managers from meta managers). + * The client goes back to a meta manager on authentication failure. + * The client prints to stdout the redirections it gets. This behavior may be + disabled by setting the XRD_PRINTREDIRECTS environment variable to 0, or, + from C++ by saying: EnvPutInt( NAME_PRINT_REDIRECTS, 0 ) + * Set $HOST value for possible copycmd substitution. + * Phase 1 to allow for redirection monitoring. Add rbuff and redir options + to the xrootd.monitor directive. + * Add error, redirect, and delay counts to the xrootd protocol summary + statistics. + * Allow file additions/deletion to be communicated to the XrdCnsd so that is + can maintain an accurate inventory. This update adds the frm.all.cnsd + directive which specifies how the information is to be commuincated. + * Enable cmsd monitoring. For now, only [meta]manager information is reported. + * Add new repstats config directive to increase reporting detail. + * New class, XrdCmsRole, to make role naming/handling consistent. + * Implement the 'cms.delay qdn' directive which allows one to tell the + meta-manager the minimum number of responses needed to satisfy a hold + delay (i.e. fast redirect). + * Accept XrdSecSSSKT envar as documented but also continue to support + XrdSecsssKT for backward compatibility. + * Allow servers to specify to the meta-manager what share of requests they + are willing to handle. Add the 'cms.sched gsdflt' and 'cms.sched gshr' + configuration directives to specify this. + * Include additional information in the protocol statistics. + * Resize some counters to prevent overflows. + * Add the 'cms.delay qdn' directive to allow better redirection control in + the future. + * Allow a plugin (notably the proxy plugin) to disable async I/O. + * Implement a general memory caching object. Currently, this will be used + by the Posix object. + * Allow optional memory caching when using the Posix library. This is + primarily used by the proxy object to reduce trips to a data server when + small blocks are accessed via the proxy server. This requires + configuration using the new 'pss.memcache' directive. + * Finally implement adding authentication information to the user monitoring + record (requested by Matevz Tadel, CMS). This adds a new generic option, + auth, to the xrootd.monitor directive. It needs to be specified for the + authentication information to be added. This keeps backward compatibility. + * Add a new method, chksum, to the standard filesystem interface. + * Integrate checksums into the logical filesystem layer implementation. + See the ofs.ckslib directive on how to do non-default configuration. + This also added a more effecient lfn2pfn() method to the storage system. + * Allow native checksums to be enabled in the xrootd layer. + See the xrootd.chksum directive on how to do this. + * Add checksum management to the frm_admin command. + * Allow XrdOucProg to dispatch a local program as well as a process. + * Allow a line to be insrerted into an XrdOucStream managed stream. + * Implement native checksums usable stand-alone or as plugins. Three digests + are supported: adler32, crc32, and md5. An additional digest can be added + via a plugin. Also, the native digests can be over-ridden via a plugin. + * In XrdSecgsi, new interface for the authorization plug-in which has now full + access to the XrdSecEntity object, with the possibility to fill/modify all the + fields according to the proxy chain. The plug-in is now called at the end of + the all process, after a successful handshake and DN-username mapping. + Implementations must contain three extern C functions; see the dummy example + provided in src/XrdSecgsi/XrdSecgsiAuthzFunDN.cc. + See also the header of XrdSecProtocolgsi::LoadAuthzFun. + * In XrdCryptosslgsiAux, add function to extract the VOMS attributes; can be + used in authz plug-ins. + * In XrdSecgsi, add possibility to extract the VOMS attributes and save them + in the XrdSecEntity. New switch '-vomsat:0/1 [1]'. + * In 'xrdgsiproxy info' show also the VOMS attributes, if present. + * Automatically build the RPM for the xrootd user when an OSG build is detected + and add fedora > 15 init scripts dependencies + ++ **Major bug fixes** + * Do not close the loger's shadow file descriptor when backgrounding as + this may cause random crashes later on. + * Avoid SEGV by setting network pointer prior to loading the 1st protocol. + * Enforce r/o path during mkdir operations. + * Avoid segv when initializing the finder on a multi-core machine. + * Fix incorrect lock handling for multiple waiters. + * Fix possible deadlocks in XrdSutCache preventing the pwd security module + to work correctly + ++ **Minor bug fixes** + * Properly handle the case when a site has an excessive number of groups + assignments. + * Prevent the response to a query from being truncated on the client side. + * Report readv information in the detailed monitoring stream. + * Correct default settings due to feature interactions after the fact. Now, + oss.defaults acts as if the setting were actually specified via oss.export. + * Actually use the N2N library of specified or implied via pss.localroot + for proxy server interactions withthe origin (required for Atlas T2). + * Use re-enterant versions of getpwuid() and getpwgid(). This is need for + FUSE. + * Correct bad english in a few error messages. + * Set correct checksum length when converting ASCII to binary. + * Allow the sss protocol to work for multi-homed hosts. + * Correct definition of AtomicISM that caused the maximum link count to + never be updated in the statistics. + * Apply N2N mapping to source path when relocating the file. + * Report correct port when locate is directly issued to a data server + (before it was being reported as 0). + * Make the default file system a pointer to a dynamic instance of XrdOfs + instead of a global static (i.e. the Andreas Peters patch). This makes + writing an ofs plugin easier. + * Fix the RPM uninstall scriptlets incorrectly invoking /sbin/ldconfig. + * Install XrdOlbMonPerf and netchk tools. + * Fix a bug preventing the core of authentication errors to be logged to clients + * In the krb5 security plugin, define KRB5CCNAME to point to the credential + cache file /tmp/krb5cc_ only if this file exists and is readable. + Solves an issue with credentials cached in memory (API::n). + * Fix array deletion mismatches reported by cppcheck (from D. Volgyes) + * Make sure that loading of XrdSecgsi.so fails if either the GMAPFun or the + AuthzFun plug-ins fail to load. + ++ **Miscellaneous** + * Drop Windows support. + * Code cleanup: remove XrdTokenAuthzOfs, simple tests, broken utilities, + the gridftp code, krb4 and secssl plugins, obsolete documentation files + * Make the loadable module extensions configurable depending on the platform + (so on Linux and Solaris, dylib on MacOs) + * Add new XrdVNUMBER macro. + * Clean up the conditional compilation macros. + * Remove compression related attributes (compchk, ssdec) and directives + (compdetect) as they were never used nor fully implemented. + * Remove the userprty directive. It was deprecated and never specified. + * Refactor PosixPreeload and Posix libraries to prevent split initialization + of the preload library which will cause failures on certain systems. + * Provide automatic proxy checksum defaults when role is set to proxy. + * Remove all references via extern statements to object instances. This + only applies to the Xrd package. + * Do not echo lines qualified by an in-line if when the if fails. + * Remove the old "redirect" directive. It has passed its prime. + * Remove back references to symbols defined in XrdXrootd package used by + the cms client to allow for clean shared library builds. + * Remove externs to XrdSecGetProtocol and XrdSecGetService from + XrdSecInterface.hh to avoid having undefined references just because the + include file was included somewhere. + * Rename XrdNetDNS to XrdSysDNS to avoid cross-dependencies. This means that all + plug-in developers will need to do the same as XrdNetDNS no longer exists. + * Split XrdFrm into XrdFrm and XrdFrc. This prevents cross-dependencies in + packages that use the File Residency Manager. + +------------- +Version 3.0.5 +------------- + ++ **Major bug fixes** + * Avoid stage failures when target file exists in purgeable or writable space. + * Make sure all the threads are joined when closing a physical connection. + * Fix free/delete mismatch in XrdSecProtocolgsi et al. + ++ **Minor bug fixes** + * Remove old async shutdown workaround patch introduced in Linux 2.3. The + problem has been since fixed and the solution now causes problems. + * Install the netchk tool + +------------- +Version 3.0.4 +------------- + ++ **New features** + * xrdcp now has -version parameter + * xrdcp automatically ads the oss.asize hint to the url opaque data. + This functionality may be disabled by setting the XrdCpSizeHint + variable to 0 (XRD_XRDCPSIZEHIN in the shell). + * The client will try to resolve the server hostname on every retry to + enable DNS failovers. + * RPM: devel package split into libs-devel, client-devel and server-devel + * XrootdFS: all paramenters can be passed via command line, add -h. + * Allow a plugin (notably the proxy plugin) to disable async I/O. + * New class XrdSysRWLock interfacing the pthread_rwlock functionality + * In XrdSecEntity: Add new fields 'creds' and 'credslen' to be filled + with the raw client credentials + * In XrdSutCache: Use XrdSysRWLock to coordinate concurrent access to + the cache + * In XrdSecgsi: + + - Add option to have Entity.name filled with the plain DN, instead of + the DN hash, when no mapping is requested or found. + + - Enable cache also for authz mapping results. + + - Do not require the existence of a grid-mapfile if gmapopt=2 and there is at least + a gmapfun or an authzfun defined. + + - Add example of mapping function allowing to match parts of the DN + + - Extend existing option 'authzpxy' to allow exporting the incoming client credentials in + XrdSecEntity. + ++ **Major bug fixes** + * Async write errors are now being properly caught and reacted to. + XrdClient::Close will now fail if it cannot recover from async + write errors. + * xrdcp prints an error message and returns failure to the shell + when some of the write requests it issues fail. + * libXrdPosixPreload now builds with autotools and is included into + the xrootd-client RPM + * RPM: FFS moved from libs to client + * Properly parse oss.asize. This because a major problem when xrdcp started + adding this to the url which causes the copy to fail. + * Spin connection portion of proxy initialization to a background thread. + This prevents init.d hangs when a redirector is not available. + ++ **Minor bug fixes** + * Test for 64-bit atomics instead 32-bit ones. Fixes build on 32-bit PowerPC. + * RPM: xrootd-fuse now depends on fuse + * Take correctly into accoutn summer time in calculating the time left for + a proxy validity + * Add support for Ubuntu 11 which uses the directory /usr/lib/`dpkg-architecture + -qDEB_HOST_MULTIARCH` to store platform dependent libraries. + +------------- +Version 3.0.3 +------------- + ++ **New features** + * Change configure.classic to handle various versions of processors in a + more sane way (this fixes several Solaris issues and atomics for i686). + * Add fwdwait option to cms.request directive to allow pacing of forwarded + requests (off by default). + * Use native memory synchronization primitives when available when doing + network I/O. This will eventually be extended to cover all other cases. + * Add the qdl option to the cms.delay directive to allow changing the + query window independently of the time a client is asked to wait for the + query to actually complete. + * Add 'pss.namelib' directive to allow proxies to pre-translate the lfn + for servers that cannot do so (e.g., dCache xrootd door). + * Optimize handling of shared-everything ile systems (e.g., dCache, GPFS, + DFS, Lustre, etc.) in the cmsd. + * Implement optional throttling for meta-manager requests in the cmsd. + * New cmsd directive, cms.dfs, declares that the underlying file system + is a shared-everything system (i.e., distributed file system) and allow + for optimal configuration and meta-manager throttling. + * Change the oss and fm components to use file extended attributes instead + of meta-files. This affects copy, create, reloc, rename, and unlink in the + oss layer. Migrate, purge, transfer, and most admin commands in the frm + component. The 'all.export' directive now accepts the noxattr/xattr option. + WARNING: If the migrator or purge options have been specified for any path + in the 'all.export; directive then this change requires either the the + 'oss.runmodeold' directive be added to the configuration file to provide + backward compatibility or that the name and data spaces be migrated using + the frm_admin command. See "Migrating tp Extended Attributes" manual for + detailed information and the new 'frm_admin convert' subcommand. + * Avoid physical copy if the operation can be handled using hard links. This + greatly speeds up static space token reassignment. + * Add platform independent interface to extended file attributes. + * RPM packaging and Red Hat Enterprise Linux compatible init scripts + capable of handling multiple instances of the xrootd daemons. The instances + can be defined in the /etc/sysconfig/xrootd file and then handled using standard:: + + service xrootd start|stop|... + service cmsd start|stop|... + ... + + or handled by name:: + + service xrootd start instance1 instance5 + + * New '-s' commandline option for xrootd, cmsd, frm_purged and frm_xfrd + creating a pidfile. + * xrootd, cmsd, frm_purged and frm_xfrd now return failure to the shell + when called with '-b' option (daemonization) and the daemon fails to + initialize. + * New 'EnableTCPKeepAlive' client environment option added enabling the TCP + stack keep-alive functionality for the sockets. + On Linux three addtional fine-tunning options are available: + + - TCPKeepAliveTime - interval (in seconds) between the last data packet and the first keep-alive + probe + - TCPKeepAliveInterval - interval (in seconds) between the probes + - TCPKeepAliveProbes - number of probes lost to consider the connection broken + + * New functionality handling process forking. When enabled (via the 'EnableForkHandlers' + env option) prior to a call to fork it shuts down all the xrootd connection management + facilities (including the connections themselves) and reinitializes them after the fork + both in the parent and the child process. This ensures relative fork safety provided + that all the XrdClient and XrdAdmin instances are closed when the fork function is invoked. + ++ **Major bug fixes** + * Add missing braces that caused config failure in frm_admin command. + * Account for correct path when -M value is zero in hpsscp command. + * In XrdCryptossl, fix for thread-safeness; solves random crashes observed on the + server side under high authentication frequency + * In XrdOucBonjour, fix important issue with host domain name registration, preventing + the correct domain to be posted. + ++ **Minor bug fixes** + * Correct file discovery propogation for proxy manager relative to meta-managers. + * Correct oss partition selection algorithm to further spread out file + allocation. + * Allow underscores in set/setenv variables. + * Add null byte after checksum value response. + * Move mapping of errno to xrootd error code to the protocol package where it + belongs. This also removes a cross dependency. + * Correct RetToken() behaviour in the presence of multiple spaces between tokens and + the previous call returned the remainder of the line (very obscure circumstances). + * [bug #77535] xrdcp now returns an error to the shell when it fails to copy the file + * [bug #79710] xrdcp now gracefully aborts when it encounters a corrupted local file + * Reset the transaction timeout for the Query method. + This fixes transaction timeout issues for clients doing only queries. + * Rename variable to remove conflict between it and global of the same name. + * Fix frm_admin command line option parsing so it does not trip over + subcommand options. This also fixes a SEGV in MacOS when this actually + happens. + * Enable the '-md5' option when OpenSSL is present and xrootd is built with autotools. + ++ **Documentation** + * Added man pages for: xprep, xrd, xrdcp, xrdstagetool, xrdgsiproxy + +------------- +Version 3.0.2 +------------- + ++ **Minor bug fixes** + * Fix the build on Solaris 10. + * Fix the build on SLC4. + * Fix the out-of-the-source-tree builds with autotools. + * Fix a segfault while doing a recursive copy from root:// to root://. + +------------- +Version 3.0.1 +------------- + ++ **New features** + * New application, cconfing, added to display configuration files relative to a host-program-instance. + * New application, netchk, that tests that firewalls have been correctly setup. + * New configure.classic option to allow use of stl4port library for Solaris. + * New internal feature in XrdPosix library to not shadow files with actual file descriptors (used by the proxy + service). This increases scalability. + * Allow the xrootd server to tell the client that it is a meta-manager. + * Support fo proxies generated by Globus version 4.2.1 in libXrdSecssl. + ++ **Major bug fixes** + * Change link options for xrdadler32 to not use shared libraries. The previous setup caused the command to hang + upon exit. + * Remove instance of XrdPosixXrootd from that same file. Including it disallows defaults from being changed. + ++ **Minor bug fixes** + * Fix XrdOucStream to not return ending "fi". + * Correct network option interference -- do not turn on network nodnr option should the keepalive option + be specified. + * Remove duplicate option in option table used by the proxy service. + * Compile on Solaris 11 Express using SunCC. + * Compile on Windows using MSVC++2010. diff --git a/packaging/debian/compat b/src/XrdCeph/packaging/debian/compat similarity index 100% rename from packaging/debian/compat rename to src/XrdCeph/packaging/debian/compat diff --git a/src/XrdCeph/packaging/debian/control b/src/XrdCeph/packaging/debian/control new file mode 100644 index 00000000000..86b8341286f --- /dev/null +++ b/src/XrdCeph/packaging/debian/control @@ -0,0 +1,48 @@ +Source: xrootd +Maintainer: Jozsef Makai +Section: misc +Priority: optional +Standards-Version: 3.9.3 +Build-Depends: debhelper (>= 9), cmake (>=3.3.0), zlib1g-dev, libfuse-dev, python-dev, libssl-dev, libxml2-dev, ncurses-dev, libkrb5-dev, libreadline-dev, libsystemd-dev, selinux-policy-dev, systemd +Homepage: https://github.com/xrootd/xrootd +Vcs-Git: https://github.com/xrootd/xrootd.git +Vcs-Browser: https://github.com/xrootd/xrootd + +Package: xrootd-libs +Architecture: any +Description: This package contains libraries used by the xrootd servers and clients. + +Package: xrootd-devel +Architecture: any +Depends: ${shlibs:Depends}, xrootd-libs (=${binary:Version}) +Description: This package contains header files and development libraries for xrootd development. + +Package: xrootd-client-libs +Architecture: any +Depends: ${shlibs:Depends}, xrootd-libs (=${binary:Version}) +Description: This package contains libraries used by xrootd clients. + +Package: xrootd-client-devel +Architecture: any +Depends: ${shlibs:Depends}, xrootd-devel (=${binary:Version}), xrootd-client-libs (=${binary:Version}) +Description: This package contains header files and development libraries for xrootd client development. + +Package: xrootd-client +Architecture: any +Depends: ${shlibs:Depends}, libxml2, xrootd-libs (=${binary:Version}), xrootd-client-libs (=${binary:Version}) +Description: This package contains the command line tools used to communicate with xrootd servers. + +Package: xrootd-private-devel +Architecture: any +Depends: ${shlibs:Depends}, xrootd-libs (=${binary:Version}) +Description: This package contains some private xrootd headers. The use of these headers is strongly discouraged. Backward compatibility between versions is not guaranteed for these headers. + +Package: xrootd-server-libs +Architecture: any +Depends: ${shlibs:Depends}, xrootd-libs (=${binary:Version}), xrootd-client-libs (=${binary:Version}) +Description: This package contains libraries used by xrootd servers. + +Package: xrootd-server-devel +Architecture: any +Depends: ${shlibs:Depends}, xrootd-devel (=${binary:Version}), xrootd-client-devel (=${binary:Version}), xrootd-server-libs (=${binary:Version}) +Description: This package contains header files and development libraries for xrootd server development. diff --git a/src/XrdCeph/packaging/debian/copyright b/src/XrdCeph/packaging/debian/copyright new file mode 100644 index 00000000000..8fd5621be2c --- /dev/null +++ b/src/XrdCeph/packaging/debian/copyright @@ -0,0 +1,18 @@ +"Copyright (c) 2005-2012, Board of Trustees of the Leland Stanford, Jr. University.\n" +"Produced under contract DE-AC02-76-SF00515 with the US Department of Energy. \n" +"All rights reserved. The copyright holder's institutional names may not be used to\n" +"endorse or promote products derived from this software without specific prior \n" +"written permission.\n\n" +"This file is part of the XRootD software suite. \n\n" +"XRootD is free software: you can redistribute it and/or modify it under the terms \n" +"of the GNU Lesser General Public License as published by the Free Software \n" +"Foundation, either version 3 of the License, or (at your option) any later version.\n\n" +"XRootD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n" +"without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR \n" +"PURPOSE. See the GNU Lesser General Public License for more details. \nn" +"You should have received a copy of the GNU Lesser General Public License along \n" +"with XRootD in a file called COPYING.LESSER (LGPL license) and file COPYING (GPL \n" +"license). If not, see .\n\n" +"Prior to September 2nd, 2012 the XRootD software suite was licensed under a\n" +"modified BSD license (see file COPYING.BSD). This applies to all code that\n" +"was in the XRootD git repository prior to that date.\n" diff --git a/packaging/debian/rules b/src/XrdCeph/packaging/debian/rules similarity index 55% rename from packaging/debian/rules rename to src/XrdCeph/packaging/debian/rules index 44203c03974..a89c9970c54 100755 --- a/packaging/debian/rules +++ b/src/XrdCeph/packaging/debian/rules @@ -1,15 +1,12 @@ #!/usr/bin/make -f -export PYBUILD_NAME=xrootd -# --install-layout deb %: - dh $@ --builddirectory=build --destdir=deb_packages --with python3 + dh $@ --builddirectory=build --destdir=deb_packages override_dh_auto_configure: - dh_auto_configure -- -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_LIBDIR=lib/$(shell dpkg-architecture -qDEB_HOST_MULTIARCH) -DPython_EXECUTABLE=/usr/bin/python3 -DCMAKE_SKIP_INSTALL_RPATH=ON -DENABLE_XRDCLHTTP=TRUE + dh_auto_configure -- -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_LIBDIR=lib/$(shell dpkg-architecture -qDEB_HOST_MULTIARCH) override_dh_install: install -D -m 644 packaging/common/client.conf deb_packages/etc/xrootd/client.conf install -D -m 644 packaging/common/client-plugin.conf.example deb_packages/etc/xrootd/client.plugins.d/client-plugin.conf.example - install -D -m 644 packaging/common/http.client.conf.example deb_packages/etc/xrootd/client.plugins.d/xrdcl-http-plugin.conf dh_install --sourcedir=deb_packages diff --git a/src/XrdCeph/packaging/debian/source/format b/src/XrdCeph/packaging/debian/source/format new file mode 100644 index 00000000000..163aaf8d82b --- /dev/null +++ b/src/XrdCeph/packaging/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/packaging/debian/libxrootd-client-dev.install b/src/XrdCeph/packaging/debian/xrootd-client-devel.install similarity index 76% rename from packaging/debian/libxrootd-client-dev.install rename to src/XrdCeph/packaging/debian/xrootd-client-devel.install index 73c00fd23ab..321cbc0430c 100644 --- a/packaging/debian/libxrootd-client-dev.install +++ b/src/XrdCeph/packaging/debian/xrootd-client-devel.install @@ -1,7 +1,9 @@ usr/bin/xrdgsitest usr/lib/*/libXrdCl.so +usr/lib/*/libXrdClient.so usr/lib/*/libXrdFfs.so usr/lib/*/libXrdPosix.so usr/share/man/man1/xrdgsitest.1* usr/include/xrootd/XrdCl +usr/include/xrootd/XrdClient usr/include/xrootd/XrdPosix diff --git a/src/XrdCeph/packaging/debian/xrootd-client-libs.install b/src/XrdCeph/packaging/debian/xrootd-client-libs.install new file mode 100644 index 00000000000..a8a45bfa66c --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-client-libs.install @@ -0,0 +1,8 @@ +usr/lib/*/libXrdCl.so.2* +usr/lib/*/libXrdClient.so.2* +usr/lib/*/libXrdFfs.so.2* +usr/lib/*/libXrdPosix.so.2* +usr/lib/*/libXrdPosixPreload.so.1* +usr/lib/*/libXrdPosixPreload.so +etc/xrootd/client.plugins.d/client-plugin.conf.example +etc/xrootd/client.conf diff --git a/src/XrdCeph/packaging/debian/xrootd-client-libs.postinst b/src/XrdCeph/packaging/debian/xrootd-client-libs.postinst new file mode 100644 index 00000000000..2983b531b1f --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-client-libs.postinst @@ -0,0 +1,3 @@ +#!/bin/bash + +ldconfig diff --git a/src/XrdCeph/packaging/debian/xrootd-client-libs.postrm b/src/XrdCeph/packaging/debian/xrootd-client-libs.postrm new file mode 100644 index 00000000000..2983b531b1f --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-client-libs.postrm @@ -0,0 +1,3 @@ +#!/bin/bash + +ldconfig diff --git a/src/XrdCeph/packaging/debian/xrootd-client.install b/src/XrdCeph/packaging/debian/xrootd-client.install new file mode 100644 index 00000000000..d4ca23ae269 --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-client.install @@ -0,0 +1,18 @@ +usr/bin/xprep +usr/bin/xrd +usr/bin/xrdadler32 +usr/bin/xrdcopy +usr/bin/xrdcp +usr/bin/xrdcp-old +usr/bin/xrdfs +usr/bin/xrdgsiproxy +usr/bin/xrdstagetool +usr/share/man/man1/xprep.1* +usr/share/man/man1/xrd.1* +usr/share/man/man1/xrdadler32.1* +usr/share/man/man1/xrdcopy.1* +usr/share/man/man1/xrdcp.1* +usr/share/man/man1/xrdcp-old.1* +usr/share/man/man1/xrdfs.1* +usr/share/man/man1/xrdgsiproxy.1* +usr/share/man/man1/xrdstagetool.1* diff --git a/packaging/debian/libxrootd-dev.install b/src/XrdCeph/packaging/debian/xrootd-devel.install similarity index 90% rename from packaging/debian/libxrootd-dev.install rename to src/XrdCeph/packaging/debian/xrootd-devel.install index c9241223538..8ce0c9c9a9e 100644 --- a/packaging/debian/libxrootd-dev.install +++ b/src/XrdCeph/packaging/debian/xrootd-devel.install @@ -13,4 +13,3 @@ usr/lib/*/libXrdCrypto.so usr/lib/*/libXrdCryptoLite.so usr/lib/*/libXrdUtils.so usr/lib/*/libXrdXml.so -usr/share/xrootd/cmake/XRootDConfig.cmake diff --git a/src/XrdCeph/packaging/debian/xrootd-libs.install b/src/XrdCeph/packaging/debian/xrootd-libs.install new file mode 100644 index 00000000000..8faf68a3046 --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-libs.install @@ -0,0 +1,9 @@ +usr/lib/*/libXrdAppUtils.so.1* +usr/lib/*/libXrdClProxyPlugin-4.so +usr/lib/*/libXrdCks*-4.so +usr/lib/*/libXrdCrypto.so.1* +usr/lib/*/libXrdCryptoLite.so.1* +usr/lib/*/libXrdCryptossl-4.so +usr/lib/*/libXrdSec*-4.so +usr/lib/*/libXrdUtils.so.* +usr/lib/*/libXrdXml.so.* diff --git a/src/XrdCeph/packaging/debian/xrootd-libs.postinst b/src/XrdCeph/packaging/debian/xrootd-libs.postinst new file mode 100644 index 00000000000..2983b531b1f --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-libs.postinst @@ -0,0 +1,3 @@ +#!/bin/bash + +ldconfig diff --git a/src/XrdCeph/packaging/debian/xrootd-libs.postrm b/src/XrdCeph/packaging/debian/xrootd-libs.postrm new file mode 100644 index 00000000000..2983b531b1f --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-libs.postrm @@ -0,0 +1,3 @@ +#!/bin/bash + +ldconfig diff --git a/packaging/debian/libxrootd-private-dev.install b/src/XrdCeph/packaging/debian/xrootd-private-devel.install similarity index 100% rename from packaging/debian/libxrootd-private-dev.install rename to src/XrdCeph/packaging/debian/xrootd-private-devel.install diff --git a/packaging/debian/libxrootd-server-dev.install b/src/XrdCeph/packaging/debian/xrootd-server-devel.install similarity index 77% rename from packaging/debian/libxrootd-server-dev.install rename to src/XrdCeph/packaging/debian/xrootd-server-devel.install index 100447f1f6c..d1dcf33b6c5 100644 --- a/packaging/debian/libxrootd-server-dev.install +++ b/src/XrdCeph/packaging/debian/xrootd-server-devel.install @@ -1,9 +1,8 @@ usr/include/xrootd/XrdAcc usr/include/xrootd/XrdCms -usr/include/xrootd/XrdPfc +usr/include/xrootd/XrdFileCache usr/include/xrootd/XrdOss usr/include/xrootd/XrdSfs usr/include/xrootd/XrdXrootd usr/include/xrootd/XrdHttp usr/lib/*/libXrdServer.so -usr/lib/*/libXrdHttpUtils.so diff --git a/src/XrdCeph/packaging/debian/xrootd-server-libs.install b/src/XrdCeph/packaging/debian/xrootd-server-libs.install new file mode 100644 index 00000000000..de9c8be9e28 --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-server-libs.install @@ -0,0 +1,14 @@ +usr/lib/*/libXrdBwm-4.so +usr/lib/*/libXrdPss-4.so +usr/lib/*/libXrdXrootd-4.so +usr/lib/*/libXrdFileCache-4.so +usr/lib/*/libXrdBlacklistDecision-4.so +usr/lib/*/libXrdHttp-4.so +usr/lib/*/libXrdN2No2p-4.so +usr/lib/*/libXrdOssSIgpfsT-4.so +usr/lib/*/libXrdServer.so.* +usr/lib/*/libXrdSsi-4.so +usr/lib/*/libXrdSsiLib.so.* +usr/lib/*/libXrdSsiLog-4.so +usr/lib/*/libXrdSsiShMap.so.* +usr/lib/*/libXrdThrottle-4.so diff --git a/src/XrdCeph/packaging/debian/xrootd-server-libs.postinst b/src/XrdCeph/packaging/debian/xrootd-server-libs.postinst new file mode 100644 index 00000000000..2983b531b1f --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-server-libs.postinst @@ -0,0 +1,3 @@ +#!/bin/bash + +ldconfig diff --git a/src/XrdCeph/packaging/debian/xrootd-server-libs.postrm b/src/XrdCeph/packaging/debian/xrootd-server-libs.postrm new file mode 100644 index 00000000000..2983b531b1f --- /dev/null +++ b/src/XrdCeph/packaging/debian/xrootd-server-libs.postrm @@ -0,0 +1,3 @@ +#!/bin/bash + +ldconfig diff --git a/src/XrdCeph/packaging/debian_scripts/publish_debian_cern.sh b/src/XrdCeph/packaging/debian_scripts/publish_debian_cern.sh new file mode 100755 index 00000000000..204914cec25 --- /dev/null +++ b/src/XrdCeph/packaging/debian_scripts/publish_debian_cern.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +#------------------------------------------------------------------------------- +# Publish debian artifacts on CERN Gitlab CI +# Author: Jozsef Makai (11.08.2017) +#------------------------------------------------------------------------------- + +set -e + +comp=$1 +prefix=/eos/project/s/storage-ci/www/debian/xrootd + +for dist in artful xenial; do + echo "Publishing for $dist"; + path=$prefix/pool/$dist/$comp/x/xrootd/; + mkdir -p $path; + if [[ "$comp" == "master" ]]; then find ${path} -type f -name '*deb' -delete; fi + cp $dist/*deb $path; + mkdir -p $prefix/dists/$dist/$comp/binary-amd64/; + (cd $prefix && apt-ftparchive --arch amd64 packages pool/$dist/$comp/ > dists/$dist/$comp/binary-amd64/Packages); + gzip -c $prefix/dists/$dist/$comp/binary-amd64/Packages > $prefix/dists/$dist/$comp/binary-amd64/Packages.gz; + components=$(find $prefix/dists/$dist/ -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | tr '\n' ' ') + if [ -e $prefix/dists/$dist/Release ]; then + rm $prefix/dists/$dist/Release + fi + if [ -e $prefix/dists/$dist/InRelease ]; then + rm $prefix/dists/$dist/InRelease + fi + if [ -e $prefix/dists/$dist/Release.gpg ]; then + rm $prefix/dists/$dist/Release.gpg + fi + apt-ftparchive -o APT::FTPArchive::Release::Origin=CERN -o APT::FTPArchive::Release::Label=XrootD -o APT::FTPArchive::Release::Codename=artful -o APT::FTPArchive::Release::Architectures=amd64 -o APT::FTPArchive::Release::Components="$components" release $prefix/dists/$dist/ > $prefix/dists/$dist/Release; + gpg --homedir /home/stci/ --clearsign -o $prefix/dists/$dist/InRelease $prefix/dists/$dist/Release; + gpg --homedir /home/stci/ -abs -o $prefix/dists/$dist/Release.gpg $prefix/dists/$dist/Release; +done diff --git a/src/XrdCeph/packaging/makesrpm.sh b/src/XrdCeph/packaging/makesrpm.sh new file mode 100755 index 00000000000..b74b90fd471 --- /dev/null +++ b/src/XrdCeph/packaging/makesrpm.sh @@ -0,0 +1,256 @@ +#!/bin/bash +#------------------------------------------------------------------------------- +# Create a source RPM package +# Author: Lukasz Janyst (10.03.2011) +#------------------------------------------------------------------------------- + +RCEXP='^[0-9]+\.[0-9]+\.[0-9]+\-rc.*$' +CERNEXP='^[0-9]+\.[0-9]+\.[0-9]+\-[0-9]+\.CERN.*$' + +#------------------------------------------------------------------------------- +# Find a program +#------------------------------------------------------------------------------- +function findProg() +{ + for prog in $@; do + if test -x "`which $prog 2>/dev/null`"; then + echo $prog + break + fi + done +} + +#------------------------------------------------------------------------------- +# Print help +#------------------------------------------------------------------------------- +function printHelp() +{ + echo "Usage:" 1>&2 + echo "${0} [--help] [--source PATH] [--output PATH]" 1>&2 + echo " --help prints this message" 1>&2 + echo " --source PATH specify the root of the source tree" 1>&2 + echo " defaults to ../" 1>&2 + echo " --output PATH the directory where the source rpm" 1>&2 + echo " should be stored, defaulting to ." 1>&2 + echo " --version VERSION the version provided by user" 1>&2 + echo " --define 'MACRO EXPR'" 1>&2 +} + +#------------------------------------------------------------------------------- +# Parse the commandline, if only we could use getopt... :( +#------------------------------------------------------------------------------- +SOURCEPATH="../" +OUTPUTPATH="." +PRINTHELP=0 + +while test ${#} -ne 0; do + if test x${1} = x--help; then + PRINTHELP=1 + elif test x${1} = x--source; then + if test ${#} -lt 2; then + echo "--source parameter needs an argument" 1>&2 + exit 1 + fi + SOURCEPATH=${2} + shift + elif test x${1} = x--output; then + if test ${#} -lt 2; then + echo "--output parameter needs an argument" 1>&2 + exit 1 + fi + OUTPUTPATH=${2} + shift + elif test x${1} = x--version; then + if test ${#} -lt 2; then + echo "--version parameter needs an argument" 1>&2 + exit 1 + fi + USER_VERSION="--version ${2}" + shift + elif test x${1} = x--define; then + if test ${#} -lt 2; then + echo "--define parameter needs an argument" 1>&2 + exit 1 + fi + USER_DEFINE="$USER_DEFINE --define \""${2}"\"" + shift + fi + shift +done + +if test $PRINTHELP -eq 1; then + printHelp + exit 0 +fi + +echo "[i] Working on: $SOURCEPATH" +echo "[i] Storing the output to: $OUTPUTPATH" + +#------------------------------------------------------------------------------- +# Check if the source and the output dirs +#------------------------------------------------------------------------------- +if test ! -d $SOURCEPATH -o ! -r $SOURCEPATH; then + echo "[!] Source path does not exist or is not readable" 1>&2 + exit 2 +fi + +if test ! -d $OUTPUTPATH -o ! -w $OUTPUTPATH; then + echo "[!] Output path does not exist or is not writeable" 1>&2 + exit 2 +fi + +#------------------------------------------------------------------------------- +# Check if we have all the necassary components +#------------------------------------------------------------------------------- +if test x`findProg rpmbuild` = x; then + echo "[!] Unable to find rpmbuild, aborting..." 1>&2 + exit 1 +fi + +if test x`findProg git` = x; then + echo "[!] Unable to find git, aborting..." 1>&2 + exit 1 +fi + +#------------------------------------------------------------------------------- +# Check if the source is a git repository +#------------------------------------------------------------------------------- +if test ! -d $SOURCEPATH/.git; then + echo "[!] I can only work with a git repository" 1>&2 + exit 2 +fi + +#------------------------------------------------------------------------------- +# Check the version number +#------------------------------------------------------------------------------- +if test ! -x $SOURCEPATH/genversion.sh; then + echo "[!] Unable to find the genversion script" 1>&2 + exit 3 +fi + +VERSION=`$SOURCEPATH/genversion.sh --print-only $USER_VERSION $SOURCEPATH 2>/dev/null` +if test $? -ne 0; then + echo "[!] Unable to figure out the version number" 1>&2 + exit 4 +fi + +echo "[i] Working with version: $VERSION" + +if test x${VERSION:0:1} = x"v"; then + VERSION=${VERSION:1} +fi + +#------------------------------------------------------------------------------- +# Deal with release candidates +#------------------------------------------------------------------------------- +RELEASE=1 +if test x`echo $VERSION | egrep $RCEXP` != x; then + RELEASE=0.`echo $VERSION | sed 's/.*-rc/rc/'` + VERSION=`echo $VERSION | sed 's/-rc.*//'` +fi + +#------------------------------------------------------------------------------- +# Deal with CERN releases +#------------------------------------------------------------------------------- +if test x`echo $VERSION | egrep $CERNEXP` != x; then + RELEASE=`echo $VERSION | sed 's/.*-//'` + VERSION=`echo $VERSION | sed 's/-.*\.CERN//'` +fi + +#------------------------------------------------------------------------------- +# In case of user version check if the release number has been provided +#------------------------------------------------------------------------------- +if test x"$USER_VERSION" != x; then + TMP=`echo $VERSION | sed 's#.*-##g'` + if test $TMP != $VERSION; then + RELEASE=$TMP + VERSION=`echo $VERSION | sed 's#-[^-]*$##'` + fi +fi + +VERSION=`echo $VERSION | sed 's/-/./g'` +echo "[i] RPM compliant version: $VERSION-$RELEASE" + +#------------------------------------------------------------------------------- +# Create a tempdir and copy the files there +#------------------------------------------------------------------------------- +# exit on any error +set -e + +TEMPDIR=`mktemp -d /tmp/xrootd-ceph.srpm.XXXXXXXXXX` +RPMSOURCES=$TEMPDIR/rpmbuild/SOURCES +mkdir -p $RPMSOURCES +mkdir -p $TEMPDIR/rpmbuild/SRPMS + +echo "[i] Working in: $TEMPDIR" 1>&2 + +if test -d rhel -a -r rhel; then + for i in rhel/*; do + cp $i $RPMSOURCES + done +fi + +if test -d common -a -r common; then + for i in common/*; do + cp $i $RPMSOURCES + done +fi + +#------------------------------------------------------------------------------- +# Generate the spec file +#------------------------------------------------------------------------------- +if test ! -r rhel/xrootd-ceph.spec.in; then + echo "[!] The specfile template does not exist!" 1>&2 + exit 7 +fi +cat rhel/xrootd-ceph.spec.in | sed "s/__VERSION__/$VERSION/" | \ + sed "s/__RELEASE__/$RELEASE/" > $TEMPDIR/xrootd-ceph.spec + +#------------------------------------------------------------------------------- +# Make a tarball of the latest commit on the branch +#------------------------------------------------------------------------------- +# no more exiting on error +set +e + +CWD=$PWD +cd $SOURCEPATH +COMMIT=`git log --pretty=format:"%H" -1` + +if test $? -ne 0; then + echo "[!] Unable to figure out the git commit hash" 1>&2 + exit 5 +fi + +git archive --prefix=xrootd-ceph/ --format=tar $COMMIT | gzip -9fn > \ + $RPMSOURCES/xrootd-ceph.tar.gz + +if test $? -ne 0; then + echo "[!] Unable to create the source tarball" 1>&2 + exit 6 +fi + +cd $CWD + +#------------------------------------------------------------------------------- +# Build the source RPM +#------------------------------------------------------------------------------- +echo "[i] Creating the source RPM..." + +# Dirty, dirty hack! +echo "%_sourcedir $RPMSOURCES" >> $TEMPDIR/rpmmacros +eval "rpmbuild --define \"_topdir $TEMPDIR/rpmbuild\" \ + --define \"%_sourcedir $RPMSOURCES\" \ + --define \"%_srcrpmdir %{_topdir}/SRPMS\" \ + --define \"_source_filedigest_algorithm md5\" \ + --define \"_binary_filedigest_algorithm md5\" \ + ${USER_DEFINE} \ + -bs $TEMPDIR/xrootd-ceph.spec > $TEMPDIR/log" +if test $? -ne 0; then + echo "[!] RPM creation failed" 1>&2 + exit 8 +fi + +cp $TEMPDIR/rpmbuild/SRPMS/xrootd-ceph*.src.rpm $OUTPUTPATH +rm -rf $TEMPDIR + +echo "[i] Done." diff --git a/src/XrdCeph/packaging/rhel/xrootd-ceph.spec.in b/src/XrdCeph/packaging/rhel/xrootd-ceph.spec.in new file mode 100644 index 00000000000..03b1caccebd --- /dev/null +++ b/src/XrdCeph/packaging/rhel/xrootd-ceph.spec.in @@ -0,0 +1,167 @@ +#------------------------------------------------------------------------------- +# Helper macros +#------------------------------------------------------------------------------- +%if %{?rhel:1}%{!?rhel:0} + %if %{rhel} >= 7 + %define use_systemd 1 + %else + %define use_systemd 0 + %endif +%else + %if %{?fedora}%{!?fedora:0} >= 19 + %define use_systemd 1 + %else + %define use_systemd 0 + %endif +%endif + +%if %{?fedora}%{!?fedora:0} >= 22 + %define use_libc_semaphore 1 +%else + %define use_libc_semaphore 0 +%endif + +%if %{?_with_ceph11:1}%{!?_with_ceph11:0} + %define _with_ceph 1 +%endif + +%if %{?rhel:1}%{!?rhel:0} + %if %{rhel} > 7 + %define use_cmake3 0 + %else + %define use_cmake3 1 + %endif +%else + %define use_cmake3 0 +%endif + +#------------------------------------------------------------------------------- +# Package definitions +#------------------------------------------------------------------------------- +Name: xrootd-ceph +Epoch: 1 +Version: __VERSION__ +Release: __RELEASE__%{?dist}%{?_with_clang:.clang} +Summary: CEPH plug-in for XRootD +Group: System Environment/Daemons +License: LGPLv3+ +URL: http://xrootd.org/ + +# git clone http://xrootd.org/repo/xrootd.git xrootd +# cd xrootd +# git-archive master | gzip -9 > ~/rpmbuild/SOURCES/xrootd.tgz +Source0: xrootd-ceph.tar.gz + +BuildRoot: %{_tmppath}/%{name}-root + +%if %{use_cmake3} +BuildRequires: cmake3 +%else +BuildRequires: cmake +%endif + +%if %{?_with_tests:1}%{!?_with_tests:0} +BuildRequires: cppunit-devel +%endif + +BuildRequires: librados-devel = 2:14.2.15 +BuildRequires: libradosstriper-devel = 2:14.2.15 + +%if %{?_with_clang:1}%{!?_with_clang:0} +BuildRequires: clang +%endif + +BuildRequires: xrootd-server-devel%{?_isa} = %{epoch}:%{version}-%{release} +BuildRequires: xrootd-private-devel%{?_isa} = %{epoch}:%{version}-%{release} +BuildRequires: xrootd-libs%{?_isa} = %{epoch}:%{version}-%{release} +BuildRequires: xrootd-server-libs%{?_isa} = %{epoch}:%{version}-%{release} +BuildRequires: xrootd-client-libs%{?_isa} = %{epoch}:%{version}-%{release} + +Requires: xrootd-server-libs%{?_isa} = %{epoch}:%{version}-%{release} +Requires: xrootd-client-libs%{?_isa} = %{epoch}:%{version}-%{release} +Requires: xrootd-libs%{?_isa} = %{epoch}:%{version}-%{release} + +%description +The xrootd-ceph is an OSS layer plug-in for the XRootD server for interfacing +with the Ceph storage platform. + +#------------------------------------------------------------------------------- +# Build instructions +#------------------------------------------------------------------------------- +%prep +%setup -c -n xrootd-ceph + +%build +cd xrootd-ceph + +%if %{?_with_clang:1}%{!?_with_clang:0} +export CC=clang +export CXX=clang++ +%endif + +mkdir build +pushd build + +%if %{use_cmake3} +cmake3 \ +%else +cmake \ +%endif + -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RelWithDebInfo \ +%if %{?_with_tests:1}%{!?_with_tests:0} + -DENABLE_TESTS=TRUE \ +%else + -DENABLE_TESTS=FALSE \ +%endif + ../ + +make -i VERBOSE=1 %{?_smp_mflags} +popd + +#------------------------------------------------------------------------------- +# Installation +#------------------------------------------------------------------------------- +%install +rm -rf $RPM_BUILD_ROOT + +#------------------------------------------------------------------------------- +# Install 4.x.y +#------------------------------------------------------------------------------- +pushd xrootd-ceph +pushd build +make install DESTDIR=$RPM_BUILD_ROOT +popd + +# ceph posix unversioned so +rm -f $RPM_BUILD_ROOT%{_libdir}/libXrdCephPosix.so + + +%clean +rm -rf $RPM_BUILD_ROOT + +#------------------------------------------------------------------------------- +# Files +#------------------------------------------------------------------------------- +%files +%defattr(-,root,root,-) +%{_libdir}/libXrdCeph-5.so +%{_libdir}/libXrdCephXattr-5.so +%{_libdir}/libXrdCephPosix.so* + +%if %{?_with_tests:1}%{!?_with_tests:0} +%files tests +%defattr(-,root,root,-) +%{_libdir}/libXrdCephTests*.so +%endif + +#------------------------------------------------------------------------------- +# Changelog +#------------------------------------------------------------------------------- +%changelog +* Wed Dec 16 2020 George Patargias +- updated version for librados-devel and libradosstriper-devel to 14.2.15 following the recent upgrade on external Echo gateways +- fixed version in xrootd-ceph shared libraries +* Mon Mar 02 2020 Michal Simon +- fixed RPM dependencies +* Thu Mar 08 2018 Michal Simon +- initial release diff --git a/src/XrdCeph/src/CMakeLists.txt b/src/XrdCeph/src/CMakeLists.txt new file mode 100644 index 00000000000..b044ee25fec --- /dev/null +++ b/src/XrdCeph/src/CMakeLists.txt @@ -0,0 +1,10 @@ + +#------------------------------------------------------------------------------- +# Include the subcomponents +#------------------------------------------------------------------------------- +include_directories( ${CMAKE_CURRENT_SOURCE_DIR} ) +if( XRDCEPH_SUBMODULE ) + add_compile_definitions( XRDCEPH_SUBMODULE ) +endif() +include( XrdCeph ) + diff --git a/src/XrdCeph/src/XrdCeph.cmake b/src/XrdCeph/src/XrdCeph.cmake new file mode 100644 index 00000000000..60fb1da2099 --- /dev/null +++ b/src/XrdCeph/src/XrdCeph.cmake @@ -0,0 +1,76 @@ +include_directories( ${XROOTD_INCLUDE_DIR} ) +include_directories( ${RADOS_INCLUDE_DIR} ) +include_directories( ${CMAKE_SOURCE_DIR}/src ) + + +#------------------------------------------------------------------------------- +# XrdCephPosix library version +#------------------------------------------------------------------------------- +set( XRD_CEPH_POSIX_VERSION 0.0.1 ) +set( XRD_CEPH_POSIX_SOVERSION 0 ) + +#------------------------------------------------------------------------------- +# The XrdCephPosix library +#------------------------------------------------------------------------------- +add_library( + XrdCephPosix + SHARED + XrdCeph/XrdCephPosix.cc XrdCeph/XrdCephPosix.hh ) + +# needed during the transition between ceph giant and ceph hammer +# for object listing API +set_property(SOURCE XrdCeph/XrdCephPosix.cc + PROPERTY COMPILE_FLAGS " -Wno-deprecated-declarations") + +target_link_libraries( + XrdCephPosix + PRIVATE + ${XROOTD_LIBRARIES} + ${RADOS_LIBS} ) + +set_target_properties( + XrdCephPosix + PROPERTIES + VERSION ${XRD_CEPH_POSIX_VERSION} + SOVERSION ${XRD_CEPH_POSIX_SOVERSION} ) + +#------------------------------------------------------------------------------- +# The XrdCeph module +#------------------------------------------------------------------------------- +set( LIB_XRD_CEPH XrdCeph-${PLUGIN_VERSION} ) + +add_library( + ${LIB_XRD_CEPH} + MODULE + XrdCeph/XrdCephOss.cc XrdCeph/XrdCephOss.hh + XrdCeph/XrdCephOssFile.cc XrdCeph/XrdCephOssFile.hh + XrdCeph/XrdCephOssDir.cc XrdCeph/XrdCephOssDir.hh ) + +target_link_libraries( + ${LIB_XRD_CEPH} + PRIVATE + ${XROOTD_LIBRARIES} + XrdCephPosix ) + +#------------------------------------------------------------------------------- +# The XrdCephXattr module +#------------------------------------------------------------------------------- +set( LIB_XRD_CEPH_XATTR XrdCephXattr-${PLUGIN_VERSION} ) + +add_library( + ${LIB_XRD_CEPH_XATTR} + MODULE + XrdCeph/XrdCephXAttr.cc XrdCeph/XrdCephXAttr.hh ) + +target_link_libraries( + ${LIB_XRD_CEPH_XATTR} + PRIVATE + ${XROOTD_LIBRARIES} + XrdCephPosix ) + +#------------------------------------------------------------------------------- +# Install +#------------------------------------------------------------------------------- +install( + TARGETS ${LIB_XRD_CEPH} ${LIB_XRD_CEPH_XATTR} XrdCephPosix + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) diff --git a/src/XrdCeph/src/XrdCeph/XrdCephOss.cc b/src/XrdCeph/src/XrdCeph/XrdCephOss.cc new file mode 100644 index 00000000000..5a9ada4f51b --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephOss.cc @@ -0,0 +1,259 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#include +#include +#include + +#include "XrdCeph/XrdCephPosix.hh" +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdSys/XrdSysError.hh" +#include "XrdOuc/XrdOucTrace.hh" +#include "XrdOuc/XrdOucStream.hh" +#include "XrdOuc/XrdOucName2Name.hh" +#ifdef XRDCEPH_SUBMODULE +#include "XrdOuc/XrdOucN2NLoader.hh" +#else +#include "private/XrdOuc/XrdOucN2NLoader.hh" +#endif +#include "XrdVersion.hh" +#include "XrdCeph/XrdCephOss.hh" +#include "XrdCeph/XrdCephOssDir.hh" +#include "XrdCeph/XrdCephOssFile.hh" + +XrdVERSIONINFO(XrdOssGetStorageSystem, XrdCephOss); + +XrdSysError XrdCephEroute(0); +XrdOucTrace XrdCephTrace(&XrdCephEroute); + +// log wrapping function to be used by ceph_posix interface +char g_logstring[1024]; +static void logwrapper(char *format, va_list argp) { + vsnprintf(g_logstring, 1024, format, argp); + XrdCephEroute.Say(g_logstring); +} + +/// pointer to library providing Name2Name interface. 0 be default +/// populated in case of ceph.namelib entry in the config file +/// used in XrdCephPosix +extern XrdOucName2Name *g_namelib; + +extern "C" +{ + XrdOss* + XrdOssGetStorageSystem(XrdOss* native_oss, + XrdSysLogger* lp, + const char* config_fn, + const char* parms) + { + // Do the herald thing + XrdCephEroute.SetPrefix("ceph_"); + XrdCephEroute.logger(lp); + XrdCephEroute.Say("++++++ CERN/IT-DSS XrdCeph"); + // set parameters + try { + ceph_posix_set_defaults(parms); + } catch (std::exception &e) { + XrdCephEroute.Say("CephOss loading failed with exception. Check the syntax of parameters : ", parms); + return 0; + } + // deal with logging + ceph_posix_set_logfunc(logwrapper); + return new XrdCephOss(config_fn, XrdCephEroute); + } +} + +XrdCephOss::XrdCephOss(const char *configfn, XrdSysError &Eroute) { + Configure(configfn, Eroute); +} + +XrdCephOss::~XrdCephOss() { + ceph_posix_disconnect_all(); +} + +// declared and used in XrdCephPosix.cc +extern unsigned int g_maxCephPoolIdx; +int XrdCephOss::Configure(const char *configfn, XrdSysError &Eroute) { + int NoGo = 0; + XrdOucEnv myEnv; + XrdOucStream Config(&Eroute, getenv("XRDINSTANCE"), &myEnv, "=====> "); + // If there is no config file, nothing to be done + if (configfn && *configfn) { + // Try to open the configuration file. + int cfgFD; + if ((cfgFD = open(configfn, O_RDONLY, 0)) < 0) { + Eroute.Emsg("Config", errno, "open config file", configfn); + return 1; + } + Config.Attach(cfgFD); + // Now start reading records until eof. + char *var; + while((var = Config.GetMyFirstWord())) { + if (!strncmp(var, "ceph.nbconnections", 18)) { + var = Config.GetWord(); + if (var) { + unsigned long value = strtoul(var, 0, 10); + if (value > 0 and value <= 100) { + g_maxCephPoolIdx = value; + } else { + Eroute.Emsg("Config", "Invalid value for ceph.nbconnections in config file (must be between 1 and 100)", configfn, var); + return 1; + } + } else { + Eroute.Emsg("Config", "Missing value for ceph.nbconnections in config file", configfn); + return 1; + } + } + if (!strncmp(var, "ceph.namelib", 12)) { + var = Config.GetWord(); + if (var) { + // Warn in case parameters were givne + char parms[1040]; + if (!Config.GetRest(parms, sizeof(parms)) || parms[0]) { + Eroute.Emsg("Config", "namelib parameters will be ignored"); + } + // Load name lib + XrdOucN2NLoader n2nLoader(&Eroute,configfn,NULL,NULL,NULL); + g_namelib = n2nLoader.Load(var, XrdVERSIONINFOVAR(XrdOssGetStorageSystem), NULL); + if (!g_namelib) { + Eroute.Emsg("Config", "Unable to load library given in ceph.namelib : %s", var); + } + } else { + Eroute.Emsg("Config", "Missing value for ceph.namelib in config file", configfn); + return 1; + } + } + } + + // Now check if any errors occurred during file i/o + int retc = Config.LastError(); + if (retc) { + NoGo = Eroute.Emsg("Config", -retc, "read config file", + configfn); + } + Config.Close(); + } + return NoGo; +} + +int XrdCephOss::Chmod(const char *path, mode_t mode, XrdOucEnv *envP) { + return -ENOTSUP; +} + +int XrdCephOss::Create(const char *tident, const char *path, mode_t access_mode, + XrdOucEnv &env, int Opts) { + return -ENOTSUP; +} + +int XrdCephOss::Init(XrdSysLogger *logger, const char* configFn) { return 0; } + +//SCS - lie to posix-assuming clients about directories [fixes brittleness in GFAL2] +int XrdCephOss::Mkdir(const char *path, mode_t mode, int mkpath, XrdOucEnv *envP) { + return 0; +} + +//SCS - lie to posix-assuming clients about directories [fixes brittleness in GFAL2] +int XrdCephOss::Remdir(const char *path, int Opts, XrdOucEnv *eP) { + return 0; +} + +int XrdCephOss::Rename(const char *from, + const char *to, + XrdOucEnv *eP1, + XrdOucEnv *eP2) { + return -ENOTSUP; +} + +int XrdCephOss::Stat(const char* path, + struct stat* buff, + int opts, + XrdOucEnv* env) { + try { + if (!strcmp(path, "/")) { + // special case of a stat made by the locate interface + // we intend to then list all files + memset(buff, 0, sizeof(*buff)); + buff->st_mode = S_IFDIR | 0700; + return 0; + } else { + return ceph_posix_stat(env, path, buff); + } + } catch (std::exception &e) { + XrdCephEroute.Say("stat : invalid syntax in file parameters"); + return -EINVAL; + } +} + +int XrdCephOss::StatFS(const char *path, char *buff, int &blen, XrdOucEnv *eP) { + XrdOssVSInfo sP; + int rc = StatVS(&sP, 0, 0); + if (rc) { + return rc; + } + int percentUsedSpace = (sP.Usage*100)/sP.Total; + blen = snprintf(buff, blen, "%d %lld %d %d %lld %d", + 1, sP.Free, percentUsedSpace, 0, 0LL, 0); + return XrdOssOK; +} + +int XrdCephOss::StatVS(XrdOssVSInfo *sP, const char *sname, int updt) { + int rc = ceph_posix_statfs(&(sP->Total), &(sP->Free)); + if (rc) { + return rc; + } + sP->Large = sP->Total; + sP->LFree = sP->Free; + sP->Usage = sP->Total-sP->Free; + sP->Extents = 1; + return XrdOssOK; +} + +int XrdCephOss::Truncate (const char* path, + unsigned long long size, + XrdOucEnv* env) { + try { + return ceph_posix_truncate(env, path, size); + } catch (std::exception &e) { + XrdCephEroute.Say("truncate : invalid syntax in file parameters"); + return -EINVAL; + } +} + +int XrdCephOss::Unlink(const char *path, int Opts, XrdOucEnv *env) { + try { + return ceph_posix_unlink(env, path); + } catch (std::exception &e) { + XrdCephEroute.Say("unlink : invalid syntax in file parameters"); + return -EINVAL; + } +} + +XrdOssDF* XrdCephOss::newDir(const char *tident) { + return new XrdCephOssDir(this); +} + +XrdOssDF* XrdCephOss::newFile(const char *tident) { + return new XrdCephOssFile(this); +} + diff --git a/src/XrdCeph/src/XrdCeph/XrdCephOss.hh b/src/XrdCeph/src/XrdCeph/XrdCephOss.hh new file mode 100644 index 00000000000..838030dc3db --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephOss.hh @@ -0,0 +1,76 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#ifndef __CEPH_OSS_HH__ +#define __CEPH_OSS_HH__ + +#include +#include + +//------------------------------------------------------------------------------ +//! This class implements XrdOss interface for usage with a CEPH storage. +//! It should be loaded via the ofs.osslib directive. +//! +//! This plugin is able to use any pool of ceph with any userId. +//! There are several ways to provide the pool and userId to be used for a given +//! operation. Here is the ordered list of possibilities. +//! First one defined wins : +//! - the path can be prepended with userId and pool. Syntax is : +//! [[userId@]pool:] +//! - the XrdOucEnv parameter, when existing, can have 'cephUserId' and/or +//! 'cephPool' entries +//! - the ofs.osslib directive can provide an argument with format : +//! [userID@]pool +//! - default are 'admin' and 'default' for userId and pool respectively +//! +//! Note that the definition of a default via the ofs.osslib directive may +//! clash with one used in a ofs.xattrlib directive. In case both directives +//! have a default and they are different, the behavior is not defined. +//! In case one of the two only has a default, it will be applied for both plugins. +//------------------------------------------------------------------------------ + +class XrdCephOss : public XrdOss { +public: + XrdCephOss(const char *, XrdSysError &); + virtual ~XrdCephOss(); + + int Configure(const char *, XrdSysError &); + + virtual int Chmod(const char *, mode_t mode, XrdOucEnv *eP=0); + virtual int Create(const char *, const char *, mode_t, XrdOucEnv &, int opts=0); + virtual int Init(XrdSysLogger *, const char*); + virtual int Mkdir(const char *, mode_t mode, int mkpath=0, XrdOucEnv *eP=0); + virtual int Remdir(const char *, int Opts=0, XrdOucEnv *eP=0); + virtual int Rename(const char *, const char *, XrdOucEnv *eP1=0, XrdOucEnv *eP2=0); + virtual int Stat(const char *, struct stat *, int opts=0, XrdOucEnv *eP=0); + virtual int StatFS(const char *path, char *buff, int &blen, XrdOucEnv *eP=0); + virtual int StatVS(XrdOssVSInfo *sP, const char *sname=0, int updt=0); + virtual int Truncate(const char *, unsigned long long, XrdOucEnv *eP=0); + virtual int Unlink(const char *path, int Opts=0, XrdOucEnv *eP=0); + virtual XrdOssDF *newDir(const char *tident); + virtual XrdOssDF *newFile(const char *tident); + +}; + +#endif /* __CEPH_OSS_HH__ */ diff --git a/src/XrdCeph/src/XrdCeph/XrdCephOssDir.cc b/src/XrdCeph/src/XrdCeph/XrdCephOssDir.cc new file mode 100644 index 00000000000..6743edc5e47 --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephOssDir.cc @@ -0,0 +1,54 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#include "XrdCeph/XrdCephPosix.hh" +#include "XrdCeph/XrdCephOssDir.hh" +#include "XrdSys/XrdSysError.hh" +#include "XrdOuc/XrdOucTrace.hh" + +extern XrdSysError XrdCephEroute; + +XrdCephOssDir::XrdCephOssDir(XrdCephOss *cephOss) : m_dirp(0), m_cephOss(cephOss) {} + +int XrdCephOssDir::Opendir(const char *path, XrdOucEnv &env) { + try { + m_dirp = ceph_posix_opendir(&env, path); + if (0 == m_dirp) { + return -errno; + } + return XrdOssOK; + } catch (std::exception &e) { + XrdCephEroute.Say("opendir : invalid syntax in file parameters"); + return -EINVAL; + } +} + +int XrdCephOssDir::Close(long long *retsz) { + ceph_posix_closedir(m_dirp); + return XrdOssOK; +} + +int XrdCephOssDir::Readdir(char *buff, int blen) { + return ceph_posix_readdir(m_dirp, buff, blen); +} diff --git a/src/XrdCeph/src/XrdCeph/XrdCephOssDir.hh b/src/XrdCeph/src/XrdCeph/XrdCephOssDir.hh new file mode 100644 index 00000000000..ba94296d3d3 --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephOssDir.hh @@ -0,0 +1,72 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#ifndef __XRD_CEPH_OSS_DIR_HH__ +#define __XRD_CEPH_OSS_DIR_HH__ + +#include "XrdOss/XrdOss.hh" +#include "XrdCeph/XrdCephOss.hh" + +//------------------------------------------------------------------------------ +//! This class implements XrdOssDF interface for usage with a CEPH storage. +//! It has a very restricted usage as the only valid path for opendir is '/'. +//! The reason is that ceph is an object store where you can only list all +//! objects, and that has no notion of hierarchy +//! +//! This plugin is able to use any pool of ceph with any userId. +//! There are several ways to provide the pool and userId to be used for a given +//! operation. Here is the ordered list of possibilities. +//! First one defined wins : +//! - the path can be prepended with userId and pool. Syntax is : +//! [[userId@]pool:] +//! - the XrdOucEnv parameter, when existing, can have 'cephUserId' and/or +//! 'cephPool' entries +//! - the ofs.osslib directive can provide an argument with format : +//! [userID@]pool +//! - default are 'admin' and 'default' for userId and pool respectively +//! +//! Note that the definition of a default via the ofs.osslib directive may +//! clash with one used in a ofs.xattrlib directive. In case both directives +//! have a default and they are different, the behavior is not defined. +//! In case one of the two only has a default, it will be applied for both plugins. +//------------------------------------------------------------------------------ + +class XrdCephOssDir : public XrdOssDF { + +public: + + XrdCephOssDir(XrdCephOss *cephoss); + virtual ~XrdCephOssDir() {}; + virtual int Opendir(const char *, XrdOucEnv &); + virtual int Readdir(char *buff, int blen); + virtual int Close(long long *retsz=0); + +private: + + DIR *m_dirp; + XrdCephOss *m_cephOss; + +}; + +#endif /* __XRD_CEPH_OSS_DIR_HH__ */ diff --git a/src/XrdCeph/src/XrdCeph/XrdCephOssFile.cc b/src/XrdCeph/src/XrdCeph/XrdCephOssFile.cc new file mode 100644 index 00000000000..3a0a63f47c7 --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephOssFile.cc @@ -0,0 +1,101 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#include +#include + +#include "XrdCeph/XrdCephPosix.hh" +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdSys/XrdSysError.hh" +#include "XrdOuc/XrdOucTrace.hh" +#include "XrdSfs/XrdSfsAio.hh" + +#include "XrdCeph/XrdCephOssFile.hh" +#include "XrdCeph/XrdCephOss.hh" + +extern XrdSysError XrdCephEroute; + +XrdCephOssFile::XrdCephOssFile(XrdCephOss *cephOss) : m_fd(-1), m_cephOss(cephOss) {} + +int XrdCephOssFile::Open(const char *path, int flags, mode_t mode, XrdOucEnv &env) { + try { + int rc = ceph_posix_open(&env, path, flags, mode); + if (rc < 0) return rc; + m_fd = rc; + return XrdOssOK; + } catch (std::exception &e) { + XrdCephEroute.Say("open : invalid syntax in file parameters"); + return -EINVAL; + } +} + +int XrdCephOssFile::Close(long long *retsz) { + return ceph_posix_close(m_fd); +} + +ssize_t XrdCephOssFile::Read(off_t offset, size_t blen) { + return XrdOssOK; +} + +ssize_t XrdCephOssFile::Read(void *buff, off_t offset, size_t blen) { + return ceph_posix_pread(m_fd, buff, blen, offset); +} + +static void aioReadCallback(XrdSfsAio *aiop, size_t rc) { + aiop->Result = rc; + aiop->doneRead(); +} + +int XrdCephOssFile::Read(XrdSfsAio *aiop) { + return ceph_aio_read(m_fd, aiop, aioReadCallback); +} + +ssize_t XrdCephOssFile::ReadRaw(void *buff, off_t offset, size_t blen) { + return Read(buff, offset, blen); +} + +int XrdCephOssFile::Fstat(struct stat *buff) { + return ceph_posix_fstat(m_fd, buff); +} + +ssize_t XrdCephOssFile::Write(const void *buff, off_t offset, size_t blen) { + return ceph_posix_pwrite(m_fd, buff, blen, offset); +} + +static void aioWriteCallback(XrdSfsAio *aiop, size_t rc) { + aiop->Result = rc; + aiop->doneWrite(); +} + +int XrdCephOssFile::Write(XrdSfsAio *aiop) { + return ceph_aio_write(m_fd, aiop, aioWriteCallback); +} + +int XrdCephOssFile::Fsync() { + return ceph_posix_fsync(m_fd); +} + +int XrdCephOssFile::Ftruncate(unsigned long long len) { + return ceph_posix_ftruncate(m_fd, len); +} diff --git a/src/XrdCeph/src/XrdCeph/XrdCephOssFile.hh b/src/XrdCeph/src/XrdCeph/XrdCephOssFile.hh new file mode 100644 index 00000000000..6b0425ea951 --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephOssFile.hh @@ -0,0 +1,77 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#ifndef __XRD_CEPH_OSS_FILE_HH__ +#define __XRD_CEPH_OSS_FILE_HH__ + +#include "XrdOss/XrdOss.hh" +#include "XrdCeph/XrdCephOss.hh" + +//------------------------------------------------------------------------------ +//! This class implements XrdOssDF interface for usage with a CEPH storage. +//! +//! This plugin is able to use any pool of ceph with any userId. +//! There are several ways to provide the pool and userId to be used for a given +//! operation. Here is the ordered list of possibilities. +//! First one defined wins : +//! - the path can be prepended with userId and pool. Syntax is : +//! [[userId@]pool:] +//! - the XrdOucEnv parameter, when existing, can have 'cephUserId' and/or +//! 'cephPool' entries +//! - the ofs.osslib directive can provide an argument with format : +//! [userID@]pool +//! - default are 'admin' and 'default' for userId and pool respectively +//! +//! Note that the definition of a default via the ofs.osslib directive may +//! clash with one used in a ofs.xattrlib directive. In case both directives +//! have a default and they are different, the behavior is not defined. +//! In case one of the two only has a default, it will be applied for both plugins. +//------------------------------------------------------------------------------ + +class XrdCephOssFile : public XrdOssDF { + +public: + + XrdCephOssFile(XrdCephOss *cephoss); + virtual ~XrdCephOssFile() {}; + virtual int Open(const char *path, int flags, mode_t mode, XrdOucEnv &env); + virtual int Close(long long *retsz=0); + virtual ssize_t Read(off_t offset, size_t blen); + virtual ssize_t Read(void *buff, off_t offset, size_t blen); + virtual int Read(XrdSfsAio *aiop); + virtual ssize_t ReadRaw(void *, off_t, size_t); + virtual int Fstat(struct stat *buff); + virtual ssize_t Write(const void *buff, off_t offset, size_t blen); + virtual int Write(XrdSfsAio *aiop); + virtual int Fsync(void); + virtual int Ftruncate(unsigned long long); + +private: + + int m_fd; + XrdCephOss *m_cephOss; + +}; + +#endif /* __XRD_CEPH_OSS_FILE_HH__ */ diff --git a/src/XrdCeph/src/XrdCeph/XrdCephPosix.cc b/src/XrdCeph/src/XrdCeph/XrdCephPosix.cc new file mode 100644 index 00000000000..2e31e528774 --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephPosix.cc @@ -0,0 +1,1323 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +/* + * This interface provides wrapper methods for using ceph through a POSIX API. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "XrdSfs/XrdSfsAio.hh" +#include "XrdSys/XrdSysPthread.hh" +#include "XrdOuc/XrdOucName2Name.hh" +#include "XrdSys/XrdSysPlatform.hh" + +#include "XrdCeph/XrdCephPosix.hh" + +/// small structs to store file metadata +struct CephFile { + std::string name; + std::string pool; + std::string userId; + unsigned int nbStripes; + unsigned long long stripeUnit; + unsigned long long objectSize; +}; + +struct CephFileRef : CephFile { + int flags; + mode_t mode; + uint64_t offset; + // This mutex protects against parallel updates of the stats. + XrdSysMutex statsMutex; + uint64_t maxOffsetWritten; + uint64_t bytesAsyncWritePending; + uint64_t bytesWritten; + unsigned rdcount; + unsigned wrcount; + unsigned asyncRdStartCount; + unsigned asyncRdCompletionCount; + unsigned asyncWrStartCount; + unsigned asyncWrCompletionCount; + ::timeval lastAsyncSubmission; + double longestAsyncWriteTime; + double longestCallbackInvocation; +}; + +/// small struct for directory listing +struct DirIterator { + librados::NObjectIterator m_iterator; + librados::IoCtx *m_ioctx; +}; + +/// small struct for aio API callbacks +struct AioArgs { + AioArgs(XrdSfsAio* a, AioCB *b, size_t n, int _fd, ceph::bufferlist *_bl=0) : + aiop(a), callback(b), nbBytes(n), fd(_fd), bl(_bl) { ::gettimeofday(&startTime, nullptr); } + XrdSfsAio* aiop; + AioCB *callback; + size_t nbBytes; + int fd; + ::timeval startTime; + ceph::bufferlist *bl; +}; + +/// global variables holding stripers/ioCtxs/cluster objects +/// Note that we have a pool of them to circumvent the limitation +/// of having a single objecter/messenger per IoCtx +typedef std::map StriperDict; +std::vector g_radosStripers; +typedef std::map IOCtxDict; +std::vector g_ioCtx; +std::vector g_cluster; +/// mutex protecting the striper and ioctx maps +XrdSysMutex g_striper_mutex; +/// index of current Striper/IoCtx to be used +unsigned int g_cephPoolIdx = 0; +/// size of the Striper/IoCtx pool, defaults to 1 +/// may be overwritten in the configuration file +/// (See XrdCephOss::configure) +unsigned int g_maxCephPoolIdx = 1; +/// pointer to library providing Name2Name interface. 0 be default +/// populated in case of ceph.namelib entry in the config file in XrdCephOss +XrdOucName2Name *g_namelib = 0; + +/// global variable holding a list of files currently opened for write +std::multiset g_filesOpenForWrite; +/// global variable holding a map of file descriptor to file reference +std::map g_fds; +/// global variable remembering the next available file descriptor +unsigned int g_nextCephFd = 0; +/// mutex protecting the map of file descriptors and the openForWrite multiset +XrdSysMutex g_fd_mutex; +/// mutex protecting initialization of ceph clusters +XrdSysMutex g_init_mutex; + +/// Accessor to next ceph pool index +/// Note that this is not thread safe, but we do not care +/// as we only want a rough load balancing +unsigned int getCephPoolIdxAndIncrease() { + if (g_radosStripers.size() == 0) { + // make sure we do not have a race condition here + XrdSysMutexHelper lock(g_init_mutex); + // double check now that we have the lock + if (g_radosStripers.size() == 0) { + // initialization phase : allocate corresponding places in the vectors + for (unsigned int i = 0; i < g_maxCephPoolIdx; i++) { + g_radosStripers.push_back(StriperDict()); + g_ioCtx.push_back(IOCtxDict()); + g_cluster.push_back(0); + } + } + } + unsigned int res = g_cephPoolIdx; + unsigned nextValue = g_cephPoolIdx+1; + if (nextValue >= g_maxCephPoolIdx) { + nextValue = 0; + } + g_cephPoolIdx = nextValue; + return res; +} + +/// check whether a file is open for write +bool isOpenForWrite(std::string& name) { + XrdSysMutexHelper lock(g_fd_mutex); + return g_filesOpenForWrite.find(name) != g_filesOpenForWrite.end(); +} + +/// look for a FileRef from its file descriptor +CephFileRef* getFileRef(int fd) { + XrdSysMutexHelper lock(g_fd_mutex); + std::map::iterator it = g_fds.find(fd); + if (it != g_fds.end()) { + // We will release the lock upon exiting this function. + // The structure here is not protected from deletion, but we trust xrootd to + // ensure close (which does the deletion) will not be called before all previous + // calls are complete (including the async ones). + return &(it->second); + } else { + return 0; + } +} + +/// deletes a FileRef from the global table of file descriptors +void deleteFileRef(int fd, const CephFileRef &fr) { + XrdSysMutexHelper lock(g_fd_mutex); + if (fr.flags & (O_WRONLY|O_RDWR)) { + g_filesOpenForWrite.erase(g_filesOpenForWrite.find(fr.name)); + } + std::map::iterator it = g_fds.find(fd); + if (it != g_fds.end()) { + g_fds.erase(it); + } +} + +/** + * inserts a new FileRef into the global table of file descriptors + * and return the associated file descriptor + */ +int insertFileRef(CephFileRef &fr) { + XrdSysMutexHelper lock(g_fd_mutex); + g_fds[g_nextCephFd] = fr; + g_nextCephFd++; + if (fr.flags & (O_WRONLY|O_RDWR)) { + g_filesOpenForWrite.insert(fr.name); + } + return g_nextCephFd-1; +} + +/// global variable containing defaults for CephFiles +CephFile g_defaultParams = { "", + "default", // default pool + "admin", // default user + 1, // default nbStripes + 4 * 1024 * 1024, // default stripeUnit : 4 MB + 4 * 1024 * 1024}; // default objectSize : 4 MB + +std::string g_defaultUserId = "admin"; +std::string g_defaultPool = "default"; + +/// global variable for the log function +static void (*g_logfunc) (char *, va_list argp) = 0; + +static void logwrapper(char* format, ...) { + if (0 == g_logfunc) return; + va_list arg; + va_start(arg, format); + (*g_logfunc)(format, arg); + va_end(arg); +} + +/// simple integer parsing, to be replaced by std::stoll when C++11 can be used +static unsigned long long int stoull(const std::string &s) { + char* end; + errno = 0; + unsigned long long int res = strtoull(s.c_str(), &end, 10); + if (0 != *end) { + throw std::invalid_argument(s); + } + if (ERANGE == errno) { + throw std::out_of_range(s); + } + return res; +} + +/// simple integer parsing, to be replaced by std::stoi when C++11 can be used +static unsigned int stoui(const std::string &s) { + char* end; + errno = 0; + unsigned long int res = strtoul(s.c_str(), &end, 10); + if (0 != *end) { + throw std::invalid_argument(s); + } + if (ERANGE == errno || res > std::numeric_limits::max()) { + throw std::out_of_range(s); + } + return (unsigned int)res; +} + +/// fills the userId of a ceph file struct from a string and an environment +/// returns position of first character after the userId +static int fillCephUserId(const std::string ¶ms, XrdOucEnv *env, CephFile &file) { + // default + file.userId = g_defaultParams.userId; + // parsing + size_t atPos = params.find('@'); + if (std::string::npos != atPos) { + file.userId = params.substr(0, atPos); + return atPos+1; + } else { + if (0 != env) { + char* cuser = env->Get("cephUserId"); + if (0 != cuser) { + file.userId = cuser; + } + } + return 0; + } +} + +/// fills the pool of a ceph file struct from a string and an environment +/// returns position of first character after the pool +static int fillCephPool(const std::string ¶ms, unsigned int offset, XrdOucEnv *env, CephFile &file) { + // default + file.pool = g_defaultParams.pool; + // parsing + size_t comPos = params.find(',', offset); + if (std::string::npos == comPos) { + if (params.size() == offset) { + if (NULL != env) { + char* cpool = env->Get("cephPool"); + if (0 != cpool) { + file.pool = cpool; + } + } + } else { + file.pool = params.substr(offset); + } + return params.size(); + } else { + file.pool = params.substr(offset, comPos-offset); + return comPos+1; + } +} + +/// fills the nbStriped of a ceph file struct from a string and an environment +/// returns position of first character after the nbStripes +// this may raise std::invalid_argument and std::out_of_range +static int fillCephNbStripes(const std::string ¶ms, unsigned int offset, XrdOucEnv *env, CephFile &file) { + // default + file.nbStripes = g_defaultParams.nbStripes; + // parsing + size_t comPos = params.find(',', offset); + if (std::string::npos == comPos) { + if (params.size() == offset) { + if (NULL != env) { + char* cNbStripes = env->Get("cephNbStripes"); + if (0 != cNbStripes) { + file.nbStripes = stoui(cNbStripes); + } + } + } else { + file.nbStripes = stoui(params.substr(offset)); + } + return params.size(); + } else { + file.nbStripes = stoui(params.substr(offset, comPos-offset)); + return comPos+1; + } +} + +/// fills the stripeUnit of a ceph file struct from a string and an environment +/// returns position of first character after the stripeUnit +// this may raise std::invalid_argument and std::out_of_range +static int fillCephStripeUnit(const std::string ¶ms, unsigned int offset, XrdOucEnv *env, CephFile &file) { + // default + file.stripeUnit = g_defaultParams.stripeUnit; + // parsing + size_t comPos = params.find(',', offset); + if (std::string::npos == comPos) { + if (params.size() == offset) { + if (NULL != env) { + char* cStripeUnit = env->Get("cephStripeUnit"); + if (0 != cStripeUnit) { + file.stripeUnit = ::stoull(cStripeUnit); + } + } + } else { + file.stripeUnit = ::stoull(params.substr(offset)); + } + return params.size(); + } else { + file.stripeUnit = ::stoull(params.substr(offset, comPos-offset)); + return comPos+1; + } +} + +/// fills the objectSize of a ceph file struct from a string and an environment +/// returns position of first character after the objectSize +// this may raise std::invalid_argument and std::out_of_range +static void fillCephObjectSize(const std::string ¶ms, unsigned int offset, XrdOucEnv *env, CephFile &file) { + // default + file.objectSize = g_defaultParams.objectSize; + // parsing + if (params.size() == offset) { + if (NULL != env) { + char* cObjectSize = env->Get("cephObjectSize"); + if (0 != cObjectSize) { + file.objectSize = ::stoull(cObjectSize); + } + } + } else { + file.objectSize = ::stoull(params.substr(offset)); + } +} + +/// fill the parameters of a ceph file struct (all but name) from a string and an environment +/// see fillCephFile for the detailed syntax +void fillCephFileParams(const std::string ¶ms, XrdOucEnv *env, CephFile &file) { + // parse the params one by one + unsigned int afterUser = fillCephUserId(params, env, file); + unsigned int afterPool = fillCephPool(params, afterUser, env, file); + unsigned int afterNbStripes = fillCephNbStripes(params, afterPool, env, file); + unsigned int afterStripeUnit = fillCephStripeUnit(params, afterNbStripes, env, file); + fillCephObjectSize(params, afterStripeUnit, env, file); +} + +/// sets the default userId, pool and file layout +/// syntax is [user@]pool[,nbStripes[,stripeUnit[,objectSize]]] +/// may throw std::invalid_argument or std::out_of_range in case of error +void ceph_posix_set_defaults(const char* value) { + if (value) { + CephFile newdefault; + fillCephFileParams(value, NULL, newdefault); + g_defaultParams = newdefault; + } +} + +/// converts a logical filename to physical one if needed +void translateFileName(std::string &physName, std::string logName){ + if (0 != g_namelib) { + char physCName[MAXPATHLEN+1]; + int retc = g_namelib->lfn2pfn(logName.c_str(), physCName, sizeof(physCName)); + if (retc) { + logwrapper((char*)"ceph_namelib : failed to translate %s using namelib plugin, using it as is", logName.c_str()); + physName = logName; + } else { + physName = physCName; + } + } else { + physName = logName; + } +} + +/// fill a ceph file struct from a path and an environment +void fillCephFile(const char *path, XrdOucEnv *env, CephFile &file) { + // Syntax of the given path is : + // [[userId@]pool[,nbStripes[,stripeUnit[,objectSize]]]:] + // for the missing parts, if env is not null the entries + // cephUserId, cephPool, cephNbStripes, cephStripeUnit, and cephObjectSize + // of env will be used. + // If env is null or no entry is found for what is missing, defaults are + // applied. These defaults are initially set to 'admin', 'default', 1, 4MB and 4MB + // but can be changed via a call to ceph_posix_set_defaults + std::string spath = path; + size_t colonPos = spath.find(':'); + if (std::string::npos == colonPos) { + // deal with name translation + translateFileName(file.name, spath); + fillCephFileParams("", env, file); + } else { + translateFileName(file.name, spath.substr(colonPos+1)); + fillCephFileParams(spath.substr(0, colonPos), env, file); + } +} + +static CephFile getCephFile(const char *path, XrdOucEnv *env) { + CephFile file; + fillCephFile(path, env, file); + return file; +} + +static CephFileRef getCephFileRef(const char *path, XrdOucEnv *env, int flags, + mode_t mode, unsigned long long offset) { + CephFileRef fr; + fillCephFile(path, env, fr); + fr.flags = flags; + fr.mode = mode; + fr.offset = 0; + fr.maxOffsetWritten = 0; + fr.bytesAsyncWritePending = 0; + fr.bytesWritten = 0; + fr.rdcount = 0; + fr.wrcount = 0; + fr.asyncRdStartCount = 0; + fr.asyncRdCompletionCount = 0; + fr.asyncWrStartCount = 0; + fr.asyncWrCompletionCount = 0; + fr.lastAsyncSubmission.tv_sec = 0; + fr.lastAsyncSubmission.tv_usec = 0; + fr.longestAsyncWriteTime = 0.0l; + fr.longestCallbackInvocation = 0.0l; + return fr; +} + +inline librados::Rados* checkAndCreateCluster(unsigned int cephPoolIdx, + std::string userId = g_defaultParams.userId) { + if (0 == g_cluster[cephPoolIdx]) { + // create connection to cluster + librados::Rados *cluster = new librados::Rados; + if (0 == cluster) { + return 0; + } + int rc = cluster->init(userId.c_str()); + if (rc) { + logwrapper((char*)"checkAndCreateCluster : cluster init failed"); + delete cluster; + return 0; + } + rc = cluster->conf_read_file(NULL); + if (rc) { + logwrapper((char*)"checkAndCreateCluster : cluster read config failed, rc = %d", rc); + cluster->shutdown(); + delete cluster; + return 0; + } + cluster->conf_parse_env(NULL); + rc = cluster->connect(); + if (rc) { + logwrapper((char*)"checkAndCreateCluster : cluster connect failed, rc = %d", rc); + cluster->shutdown(); + delete cluster; + return 0; + } + g_cluster[cephPoolIdx] = cluster; + } + return g_cluster[cephPoolIdx]; +} + +int checkAndCreateStriper(unsigned int cephPoolIdx, std::string &userAtPool, const CephFile& file) { + StriperDict &sDict = g_radosStripers[cephPoolIdx]; + StriperDict::iterator it = sDict.find(userAtPool); + if (it == sDict.end()) { + // we need to create a new radosStriper + // Get a cluster + librados::Rados* cluster = checkAndCreateCluster(cephPoolIdx, file.userId); + if (0 == cluster) { + logwrapper((char*)"checkAndCreateStriper : checkAndCreateCluster failed"); + return 0; + } + // create IoCtx for our pool + librados::IoCtx *ioctx = new librados::IoCtx; + if (0 == ioctx) { + logwrapper((char*)"checkAndCreateStriper : IoCtx instantiation failed"); + cluster->shutdown(); + delete cluster; + g_cluster[cephPoolIdx] = 0; + return 0; + } + int rc = g_cluster[cephPoolIdx]->ioctx_create(file.pool.c_str(), *ioctx); + if (rc != 0) { + logwrapper((char*)"checkAndCreateStriper : ioctx_create failed, rc = %d", rc); + cluster->shutdown(); + delete cluster; + g_cluster[cephPoolIdx] = 0; + delete ioctx; + return 0; + } + // create RadosStriper connection + libradosstriper::RadosStriper *striper = new libradosstriper::RadosStriper; + if (0 == striper) { + logwrapper((char*)"checkAndCreateStriper : RadosStriper instantiation failed"); + delete ioctx; + cluster->shutdown(); + delete cluster; + g_cluster[cephPoolIdx] = 0; + return 0; + } + rc = libradosstriper::RadosStriper::striper_create(*ioctx, striper); + if (rc != 0) { + logwrapper((char*)"checkAndCreateStriper : striper_create failed, rc = %d", rc); + delete striper; + delete ioctx; + cluster->shutdown(); + delete cluster; + g_cluster[cephPoolIdx] = 0; + return 0; + } + // setup layout + rc = striper->set_object_layout_stripe_count(file.nbStripes); + if (rc != 0) { + logwrapper((char*)"checkAndCreateStriper : invalid nbStripes %d", file.nbStripes); + delete striper; + delete ioctx; + cluster->shutdown(); + delete cluster; + g_cluster[cephPoolIdx] = 0; + return 0; + } + rc = striper->set_object_layout_stripe_unit(file.stripeUnit); + if (rc != 0) { + logwrapper((char*)"checkAndCreateStriper : invalid stripeUnit %d (must be non 0, multiple of 64K)", file.stripeUnit); + delete striper; + delete ioctx; + cluster->shutdown(); + delete cluster; + g_cluster[cephPoolIdx] = 0; + return 0; + } + rc = striper->set_object_layout_object_size(file.objectSize); + if (rc != 0) { + logwrapper((char*)"checkAndCreateStriper : invalid objectSize %d (must be non 0, multiple of stripe_unit)", file.objectSize); + delete striper; + delete ioctx; + cluster->shutdown(); + delete cluster; + g_cluster[cephPoolIdx] = 0; + return 0; + } + IOCtxDict & ioDict = g_ioCtx[cephPoolIdx]; + ioDict.insert(std::pair(userAtPool, ioctx)); + sDict.insert(std::pair + (userAtPool, striper)).first; + } + return 1; +} + +static libradosstriper::RadosStriper* getRadosStriper(const CephFile& file) { + XrdSysMutexHelper lock(g_striper_mutex); + std::stringstream ss; + ss << file.userId << '@' << file.pool << ',' << file.nbStripes << ',' + << file.stripeUnit << ',' << file.objectSize; + std::string userAtPool = ss.str(); + unsigned int cephPoolIdx = getCephPoolIdxAndIncrease(); + if (checkAndCreateStriper(cephPoolIdx, userAtPool, file) == 0) { + logwrapper((char*)"getRadosStriper : checkAndCreateStriper failed"); + return 0; + } + return g_radosStripers[cephPoolIdx][userAtPool]; +} + +static librados::IoCtx* getIoCtx(const CephFile& file) { + XrdSysMutexHelper lock(g_striper_mutex); + std::stringstream ss; + ss << file.userId << '@' << file.pool << ',' << file.nbStripes << ',' + << file.stripeUnit << ',' << file.objectSize; + std::string userAtPool = ss.str(); + unsigned int cephPoolIdx = getCephPoolIdxAndIncrease(); + if (checkAndCreateStriper(cephPoolIdx, userAtPool, file) == 0) { + return 0; + } + return g_ioCtx[cephPoolIdx][userAtPool]; +} + +void ceph_posix_disconnect_all() { + XrdSysMutexHelper lock(g_striper_mutex); + for (unsigned int i= 0; i < g_maxCephPoolIdx; i++) { + for (StriperDict::iterator it2 = g_radosStripers[i].begin(); + it2 != g_radosStripers[i].end(); + it2++) { + delete it2->second; + } + for (IOCtxDict::iterator it2 = g_ioCtx[i].begin(); + it2 != g_ioCtx[i].end(); + it2++) { + delete it2->second; + } + delete g_cluster[i]; + } + g_radosStripers.clear(); + g_ioCtx.clear(); + g_cluster.clear(); +} + +void ceph_posix_set_logfunc(void (*logfunc) (char *, va_list argp)) { + g_logfunc = logfunc; +}; + +static int ceph_posix_internal_truncate(const CephFile &file, unsigned long long size); + +/** + * * brief ceph_posix_open function opens a file for read or write + * * details This function either: + * * Opens a file for reading. If the file doesn't exist, this is an error. + * * Opens a file for writing. If the file already exists, check whether overwrite has been requested. If overwrite + * * hasn't been requested for an existing file, this is an error. + * * param env XrdOucEnv* Unused + * * param pathname const char* Specify the file to open. + * * param flags int Indicates whether reading or writing, and whether to overwrite an existing file. + * * param mode mode_t Unused + * * return int This is a file descriptor (non-negative) if the operation is successful, + * * or an error code (negative value) if the operation fails + * */ + +int ceph_posix_open(XrdOucEnv* env, const char *pathname, int flags, mode_t mode){ + + CephFileRef fr = getCephFileRef(pathname, env, flags, mode, 0); + + struct stat buf; + libradosstriper::RadosStriper *striper = getRadosStriper(fr); //Get a handle to the RADOS striper API + + if (NULL == striper) { + logwrapper((char*)"Cannot create striper"); + return -EINVAL; + } + + int rc = striper->stat(fr.name, (uint64_t*)&(buf.st_size), &(buf.st_atime)); //Get details about a file + + + bool fileExists = (rc != -ENOENT); //Make clear what condition we are testing + + if ((flags&O_ACCMODE) == O_RDONLY) { // Access mode is READ + + if (fileExists) { + int fd = insertFileRef(fr); + logwrapper((char*)"File descriptor %d associated to file %s opened in read mode", fd, pathname); + return fd; + } else { + return -ENOENT; + } + + } else { // Access mode is WRITE + if (fileExists) { + if (flags & O_TRUNC) { + int rc = ceph_posix_unlink(env, pathname); + if (rc < 0 && rc != -ENOENT) { + return rc; + } + } else { + return -EEXIST; + } + } + // At this point, we know either the target file didn't exist, or the ceph_posix_unlink above removed it + int fd = insertFileRef(fr); + logwrapper((char*)"File descriptor %d associated to file %s opened in write mode", fd, pathname); + return fd; + + } + +} + +int ceph_posix_close(int fd) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + ::timeval now; + ::gettimeofday(&now, nullptr); + XrdSysMutexHelper lock(fr->statsMutex); + double lastAsyncAge = 0.0; + // Only compute an age if the starting point was set. + if (fr->lastAsyncSubmission.tv_sec && fr->lastAsyncSubmission.tv_usec) { + lastAsyncAge = 1.0 * (now.tv_sec - fr->lastAsyncSubmission.tv_sec) + + 0.000001 * (now.tv_usec - fr->lastAsyncSubmission.tv_usec); + } + logwrapper((char*)"ceph_close: closed fd %d for file %s, read ops count %d, write ops count %d, " + "async write ops %d/%d, async pending write bytes %ld, " + "async read ops %d/%d, bytes written/max offset %ld/%ld, " + "longest async write %f, longest callback invocation %f, last async op age %f", + fd, fr->name.c_str(), fr->rdcount, fr->wrcount, + fr->asyncWrCompletionCount, fr->asyncWrStartCount, fr->bytesAsyncWritePending, + fr->asyncRdCompletionCount, fr->asyncRdStartCount, fr->bytesWritten, fr->maxOffsetWritten, + fr->longestAsyncWriteTime, fr->longestCallbackInvocation, (lastAsyncAge)); + deleteFileRef(fd, *fr); + return 0; + } else { + return -EBADF; + } +} + +static off64_t lseek_compute_offset(CephFileRef &fr, off64_t offset, int whence) { + switch (whence) { + case SEEK_SET: + fr.offset = offset; + break; + case SEEK_CUR: + fr.offset += offset; + break; + default: + return -EINVAL; + } + return fr.offset; +} + +off_t ceph_posix_lseek(int fd, off_t offset, int whence) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + logwrapper((char*)"ceph_lseek: for fd %d, offset=%lld, whence=%d", fd, offset, whence); + return (off_t)lseek_compute_offset(*fr, offset, whence); + } else { + return -EBADF; + } +} + +off64_t ceph_posix_lseek64(int fd, off64_t offset, int whence) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + logwrapper((char*)"ceph_lseek64: for fd %d, offset=%lld, whence=%d", fd, offset, whence); + return lseek_compute_offset(*fr, offset, whence); + } else { + return -EBADF; + } +} + +ssize_t ceph_posix_write(int fd, const void *buf, size_t count) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + logwrapper((char*)"ceph_write: for fd %d, count=%d", fd, count); + if ((fr->flags & (O_WRONLY|O_RDWR)) == 0) { + return -EBADF; + } + libradosstriper::RadosStriper *striper = getRadosStriper(*fr); + if (0 == striper) { + return -EINVAL; + } + ceph::bufferlist bl; + bl.append((const char*)buf, count); + int rc = striper->write(fr->name, bl, count, fr->offset); + if (rc) return rc; + fr->offset += count; + XrdSysMutexHelper lock(fr->statsMutex); + fr->wrcount++; + fr->bytesWritten+=count; + if (fr->offset) fr->maxOffsetWritten = std::max(fr->offset - 1, fr->maxOffsetWritten); + return count; + } else { + return -EBADF; + } +} + +ssize_t ceph_posix_pwrite(int fd, const void *buf, size_t count, off64_t offset) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + // TODO implement proper logging level for this plugin - this should be only debug + //logwrapper((char*)"ceph_write: for fd %d, count=%d", fd, count); + if ((fr->flags & (O_WRONLY|O_RDWR)) == 0) { + return -EBADF; + } + libradosstriper::RadosStriper *striper = getRadosStriper(*fr); + if (0 == striper) { + return -EINVAL; + } + ceph::bufferlist bl; + bl.append((const char*)buf, count); + int rc = striper->write(fr->name, bl, count, offset); + if (rc) return rc; + XrdSysMutexHelper lock(fr->statsMutex); + fr->wrcount++; + fr->bytesWritten+=count; + if (offset + count) fr->maxOffsetWritten = std::max(uint64_t(offset + count - 1), fr->maxOffsetWritten); + return count; + } else { + return -EBADF; + } +} + +static void ceph_aio_write_complete(rados_completion_t c, void *arg) { + AioArgs *awa = reinterpret_cast(arg); + size_t rc = rados_aio_get_return_value(c); + // Compute statistics before reportng to xrootd, so that a close cannot happen + // in the meantime. + CephFileRef* fr = getFileRef(awa->fd); + if (fr) { + XrdSysMutexHelper lock(fr->statsMutex); + fr->asyncWrCompletionCount++; + fr->bytesAsyncWritePending -= awa->nbBytes; + fr->bytesWritten += awa->nbBytes; + if (awa->aiop->sfsAio.aio_nbytes) + fr->maxOffsetWritten = std::max(fr->maxOffsetWritten, uint64_t(awa->aiop->sfsAio.aio_offset + awa->aiop->sfsAio.aio_nbytes - 1)); + ::timeval now; + ::gettimeofday(&now, nullptr); + double writeTime = 0.000001 * (now.tv_usec - awa->startTime.tv_usec) + 1.0 * (now.tv_sec - awa->startTime.tv_sec); + fr->longestAsyncWriteTime = std::max(fr->longestAsyncWriteTime, writeTime); + } + ::timeval before, after; + if (fr) ::gettimeofday(&before, nullptr); + awa->callback(awa->aiop, rc == 0 ? awa->nbBytes : rc); + if (fr) { + ::gettimeofday(&after, nullptr); + double callbackInvocationTime = 0.000001 * (after.tv_usec - before.tv_usec) + 1.0 * (after.tv_sec - before.tv_sec); + XrdSysMutexHelper lock(fr->statsMutex); + fr->longestCallbackInvocation = std::max(fr->longestCallbackInvocation, callbackInvocationTime); + } + delete(awa); +} + +ssize_t ceph_aio_write(int fd, XrdSfsAio *aiop, AioCB *cb) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + // get the parameters from the Xroot aio object + size_t count = aiop->sfsAio.aio_nbytes; + const char *buf = (const char*)aiop->sfsAio.aio_buf; + size_t offset = aiop->sfsAio.aio_offset; + // TODO implement proper logging level for this plugin - this should be only debug + //logwrapper((char*)"ceph_aio_write: for fd %d, count=%d", fd, count); + if ((fr->flags & (O_WRONLY|O_RDWR)) == 0) { + return -EBADF; + } + // get the striper object + libradosstriper::RadosStriper *striper = getRadosStriper(*fr); + if (0 == striper) { + return -EINVAL; + } + // prepare a bufferlist around the given buffer + ceph::bufferlist bl; + bl.append(buf, count); + // get the poolIdx to use + int cephPoolIdx = getCephPoolIdxAndIncrease(); + // Get the cluster to use + librados::Rados* cluster = checkAndCreateCluster(cephPoolIdx); + if (0 == cluster) { + return -EINVAL; + } + // prepare a ceph AioCompletion object and do async call + AioArgs *args = new AioArgs(aiop, cb, count, fd); + librados::AioCompletion *completion = + cluster->aio_create_completion(args, ceph_aio_write_complete, NULL); + // do the write + int rc = striper->aio_write(fr->name, completion, bl, count, offset); + completion->release(); + XrdSysMutexHelper lock(fr->statsMutex); + fr->asyncWrStartCount++; + ::gettimeofday(&fr->lastAsyncSubmission, nullptr); + fr->bytesAsyncWritePending+=count; + return rc; + } else { + return -EBADF; + } +} + +ssize_t ceph_posix_read(int fd, void *buf, size_t count) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + // TODO implement proper logging level for this plugin - this should be only debug + //logwrapper((char*)"ceph_read: for fd %d, count=%d", fd, count); + if ((fr->flags & O_WRONLY) != 0) { + return -EBADF; + } + libradosstriper::RadosStriper *striper = getRadosStriper(*fr); + if (0 == striper) { + return -EINVAL; + } + ceph::bufferlist bl; + int rc = striper->read(fr->name, &bl, count, fr->offset); + if (rc < 0) return rc; + bl.begin().copy(rc, (char*)buf); + XrdSysMutexHelper lock(fr->statsMutex); + fr->offset += rc; + fr->rdcount++; + return rc; + } else { + return -EBADF; + } +} + +ssize_t ceph_posix_pread(int fd, void *buf, size_t count, off64_t offset) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + // TODO implement proper logging level for this plugin - this should be only debug + //logwrapper((char*)"ceph_read: for fd %d, count=%d", fd, count); + if ((fr->flags & O_WRONLY) != 0) { + return -EBADF; + } + libradosstriper::RadosStriper *striper = getRadosStriper(*fr); + if (0 == striper) { + return -EINVAL; + } + ceph::bufferlist bl; + int rc = striper->read(fr->name, &bl, count, offset); + if (rc < 0) return rc; + bl.begin().copy(rc, (char*)buf); + XrdSysMutexHelper lock(fr->statsMutex); + fr->rdcount++; + return rc; + } else { + return -EBADF; + } +} + +static void ceph_aio_read_complete(rados_completion_t c, void *arg) { + AioArgs *awa = reinterpret_cast(arg); + size_t rc = rados_aio_get_return_value(c); + if (awa->bl) { + if (rc > 0) { + awa->bl->begin().copy(rc, (char*)awa->aiop->sfsAio.aio_buf); + } + delete awa->bl; + awa->bl = 0; + } + // Compute statistics before reportng to xrootd, so that a close cannot happen + // in the meantime. + CephFileRef* fr = getFileRef(awa->fd); + if (fr) { + XrdSysMutexHelper lock(fr->statsMutex); + fr->asyncRdCompletionCount++; + } + awa->callback(awa->aiop, rc ); + delete(awa); +} + +ssize_t ceph_aio_read(int fd, XrdSfsAio *aiop, AioCB *cb) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + // get the parameters from the Xroot aio object + size_t count = aiop->sfsAio.aio_nbytes; + size_t offset = aiop->sfsAio.aio_offset; + // TODO implement proper logging level for this plugin - this should be only debug + //logwrapper((char*)"ceph_aio_read: for fd %d, count=%d", fd, count); + if ((fr->flags & O_WRONLY) != 0) { + return -EBADF; + } + // get the striper object + libradosstriper::RadosStriper *striper = getRadosStriper(*fr); + if (0 == striper) { + return -EINVAL; + } + // prepare a bufferlist to receive data + ceph::bufferlist *bl = new ceph::bufferlist(); + // get the poolIdx to use + int cephPoolIdx = getCephPoolIdxAndIncrease(); + // Get the cluster to use + librados::Rados* cluster = checkAndCreateCluster(cephPoolIdx); + if (0 == cluster) { + return -EINVAL; + } + // prepare a ceph AioCompletion object and do async call + AioArgs *args = new AioArgs(aiop, cb, count, fd, bl); + librados::AioCompletion *completion = + cluster->aio_create_completion(args, ceph_aio_read_complete, NULL); + // do the read + int rc = striper->aio_read(fr->name, completion, bl, count, offset); + completion->release(); + XrdSysMutexHelper lock(fr->statsMutex); + fr->asyncRdStartCount++; + return rc; + } else { + return -EBADF; + } +} + +int ceph_posix_fstat(int fd, struct stat *buf) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + logwrapper((char*)"ceph_stat: fd %d", fd); + // minimal stat : only size and times are filled + // atime, mtime and ctime are set all to the same value + // mode is set arbitrarily to 0666 | S_IFREG + libradosstriper::RadosStriper *striper = getRadosStriper(*fr); + if (0 == striper) { + logwrapper((char*)"ceph_stat: getRadosStriper failed"); + return -EINVAL; + } + memset(buf, 0, sizeof(*buf)); + int rc = striper->stat(fr->name, (uint64_t*)&(buf->st_size), &(buf->st_atime)); + if (rc != 0) { + return -rc; + } + buf->st_mtime = buf->st_atime; + buf->st_ctime = buf->st_atime; + buf->st_mode = 0666 | S_IFREG; + return 0; + } else { + return -EBADF; + } +} + +int ceph_posix_stat(XrdOucEnv* env, const char *pathname, struct stat *buf) { + logwrapper((char*)"ceph_stat: %s", pathname); + // minimal stat : only size and times are filled + // atime, mtime and ctime are set all to the same value + // mode is set arbitrarily to 0666 | S_IFREG + CephFile file = getCephFile(pathname, env); + libradosstriper::RadosStriper *striper = getRadosStriper(file); + if (0 == striper) { + return -EINVAL; + } + memset(buf, 0, sizeof(*buf)); + int rc = striper->stat(file.name, (uint64_t*)&(buf->st_size), &(buf->st_atime)); + if (rc != 0) { + // for non existing file. Check that we did not open it for write recently + // in that case, we return 0 size and current time + if (-ENOENT == rc && isOpenForWrite(file.name)) { + buf->st_size = 0; + buf->st_atime = time(NULL); + } else { + return -rc; + } + } + buf->st_mtime = buf->st_atime; + buf->st_ctime = buf->st_atime; + buf->st_mode = 0666 | S_IFREG; + return 0; +} + +int ceph_posix_fsync(int fd) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + // no locking of fr as it is not used. + logwrapper((char*)"ceph_sync: fd %d", fd); + return 0; + } else { + return -EBADF; + } +} + +int ceph_posix_fcntl(int fd, int cmd, ... /* arg */ ) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + logwrapper((char*)"ceph_fcntl: fd %d cmd=%d", fd, cmd); + // minimal implementation + switch (cmd) { + case F_GETFL: + return fr->mode; + default: + return -EINVAL; + } + } else { + return -EBADF; + } +} + +static ssize_t ceph_posix_internal_getxattr(const CephFile &file, const char* name, + void* value, size_t size) { + libradosstriper::RadosStriper *striper = getRadosStriper(file); + if (0 == striper) { + return -EINVAL; + } + ceph::bufferlist bl; + int rc = striper->getxattr(file.name, name, bl); + if (rc < 0) return rc; + size_t returned_size = (size_t)rcsetxattr(file.name, name, bl); + if (rc) { + return -rc; + } + return 0; +} + +ssize_t ceph_posix_setxattr(XrdOucEnv* env, const char* path, + const char* name, const void* value, + size_t size, int flags) { + logwrapper((char*)"ceph_setxattr: path %s name=%s value=%s", path, name, value); + return ceph_posix_internal_setxattr(getCephFile(path, env), name, value, size, flags); +} + +int ceph_posix_fsetxattr(int fd, + const char* name, const void* value, + size_t size, int flags) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + logwrapper((char*)"ceph_fsetxattr: fd %d name=%s value=%s", fd, name, value); + return ceph_posix_internal_setxattr(*fr, name, value, size, flags); + } else { + return -EBADF; + } +} + +static int ceph_posix_internal_removexattr(const CephFile &file, const char* name) { + libradosstriper::RadosStriper *striper = getRadosStriper(file); + if (0 == striper) { + return -EINVAL; + } + int rc = striper->rmxattr(file.name, name); + if (rc) { + return -rc; + } + return 0; +} + +int ceph_posix_removexattr(XrdOucEnv* env, const char* path, + const char* name) { + logwrapper((char*)"ceph_removexattr: path %s name=%s", path, name); + return ceph_posix_internal_removexattr(getCephFile(path, env), name); +} + +int ceph_posix_fremovexattr(int fd, const char* name) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + logwrapper((char*)"ceph_fremovexattr: fd %d name=%s", fd, name); + return ceph_posix_internal_removexattr(*fr, name); + } else { + return -EBADF; + } +} + +static int ceph_posix_internal_listxattrs(const CephFile &file, XrdSysXAttr::AList **aPL, int getSz) { + libradosstriper::RadosStriper *striper = getRadosStriper(file); + if (0 == striper) { + return -EINVAL; + } + // call ceph + std::map attrset; + int rc = striper->getxattrs(file.name, attrset); + if (rc) { + return -rc; + } + // build result + *aPL = 0; + int maxSize = 0; + for (std::map::const_iterator it = attrset.begin(); + it != attrset.end(); + it++) { + XrdSysXAttr::AList* newItem = (XrdSysXAttr::AList*)malloc(sizeof(XrdSysXAttr::AList)+it->first.size()); + newItem->Next = *aPL; + newItem->Vlen = it->second.length(); + if (newItem->Vlen > maxSize) { + maxSize = newItem->Vlen; + } + newItem->Nlen = it->first.size(); + strncpy(newItem->Name, it->first.c_str(), newItem->Vlen+1); + *aPL = newItem; + } + if (getSz) { + return 0; + } else { + return maxSize; + } +} + +int ceph_posix_listxattrs(XrdOucEnv* env, const char* path, XrdSysXAttr::AList **aPL, int getSz) { + logwrapper((char*)"ceph_listxattrs: path %s", path); + return ceph_posix_internal_listxattrs(getCephFile(path, env), aPL, getSz); +} + +int ceph_posix_flistxattrs(int fd, XrdSysXAttr::AList **aPL, int getSz) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + logwrapper((char*)"ceph_flistxattrs: fd %d", fd); + return ceph_posix_internal_listxattrs(*fr, aPL, getSz); + } else { + return -EBADF; + } +} + +void ceph_posix_freexattrlist(XrdSysXAttr::AList *aPL) { + while (aPL) { + free(aPL->Name); + XrdSysXAttr::AList *cur = aPL; + aPL = aPL->Next; + free(cur); + } +} + +int ceph_posix_statfs(long long *totalSpace, long long *freeSpace) { + logwrapper((char*)"ceph_posix_statfs"); + // get the poolIdx to use + int cephPoolIdx = getCephPoolIdxAndIncrease(); + // Get the cluster to use + librados::Rados* cluster = checkAndCreateCluster(cephPoolIdx); + if (0 == cluster) { + return -EINVAL; + } + // call ceph stat + librados::cluster_stat_t result; + int rc = cluster->cluster_stat(result); + if (0 == rc) { + *totalSpace = result.kb * 1024; + *freeSpace = result.kb_avail * 1024; + } + return rc; +} + +static int ceph_posix_internal_truncate(const CephFile &file, unsigned long long size) { + libradosstriper::RadosStriper *striper = getRadosStriper(file); + if (0 == striper) { + return -EINVAL; + } + return striper->trunc(file.name, size); +} + +int ceph_posix_ftruncate(int fd, unsigned long long size) { + CephFileRef* fr = getFileRef(fd); + if (fr) { + logwrapper((char*)"ceph_posix_ftruncate: fd %d, size %d", fd, size); + return ceph_posix_internal_truncate(*fr, size); + } else { + return -EBADF; + } +} + +int ceph_posix_truncate(XrdOucEnv* env, const char *pathname, unsigned long long size) { + logwrapper((char*)"ceph_posix_truncate : %s", pathname); + // minimal stat : only size and times are filled + CephFile file = getCephFile(pathname, env); + return ceph_posix_internal_truncate(file, size); +} + +int ceph_posix_unlink(XrdOucEnv* env, const char *pathname) { + logwrapper((char*)"ceph_posix_unlink : %s", pathname); + // minimal stat : only size and times are filled + CephFile file = getCephFile(pathname, env); + libradosstriper::RadosStriper *striper = getRadosStriper(file); + if (0 == striper) { + return -EINVAL; + } + return striper->remove(file.name); +} + +DIR* ceph_posix_opendir(XrdOucEnv* env, const char *pathname) { + logwrapper((char*)"ceph_posix_opendir : %s", pathname); + // only accept root dir, as there is no concept of dirs in object stores + CephFile file = getCephFile(pathname, env); + if (file.name.size() != 1 || file.name[0] != '/') { + errno = -ENOENT; + return 0; + } + librados::IoCtx *ioctx = getIoCtx(file); + if (0 == ioctx) { + errno = EINVAL; + return 0; + } + DirIterator* res = new DirIterator(); + res->m_iterator = ioctx->nobjects_begin(); + res->m_ioctx = ioctx; + return (DIR*)res; +} + +int ceph_posix_readdir(DIR *dirp, char *buff, int blen) { + librados::NObjectIterator &iterator = ((DirIterator*)dirp)->m_iterator; + librados::IoCtx *ioctx = ((DirIterator*)dirp)->m_ioctx; + while (iterator->get_oid().compare(iterator->get_oid().size()-17, 17, ".0000000000000000") && + iterator != ioctx->nobjects_end()) { + iterator++; + } + if (iterator == ioctx->nobjects_end()) { + buff[0] = 0; + } else { + int l = iterator->get_oid().size()-17; + if (l < blen) blen = l; + strncpy(buff, iterator->get_oid().c_str(), blen-1); + buff[blen-1] = 0; + iterator++; + } + return 0; +} + +int ceph_posix_closedir(DIR *dirp) { + delete ((DirIterator*)dirp); + return 0; +} diff --git a/src/XrdCeph/src/XrdCeph/XrdCephPosix.hh b/src/XrdCeph/src/XrdCeph/XrdCephPosix.hh new file mode 100644 index 00000000000..77f9fbc23da --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephPosix.hh @@ -0,0 +1,77 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +/* + * This interface provides wrapper methods for using ceph through a POSIX API. + */ + +#ifndef _XRD_CEPH_POSIX_H +#define _XRD_CEPH_POSIX_H + +#include +#include +#include +#include +#include + +class XrdSfsAio; +typedef void(AioCB)(XrdSfsAio*, size_t); + +void ceph_posix_set_defaults(const char* value); +void ceph_posix_disconnect_all(); +void ceph_posix_set_logfunc(void (*logfunc) (char *, va_list argp)); +int ceph_posix_open(XrdOucEnv* env, const char *pathname, int flags, mode_t mode); +int ceph_posix_close(int fd); +off_t ceph_posix_lseek(int fd, off_t offset, int whence); +off64_t ceph_posix_lseek64(int fd, off64_t offset, int whence); +ssize_t ceph_posix_write(int fd, const void *buf, size_t count); +ssize_t ceph_posix_pwrite(int fd, const void *buf, size_t count, off64_t offset); +ssize_t ceph_aio_write(int fd, XrdSfsAio *aiop, AioCB *cb); +ssize_t ceph_posix_read(int fd, void *buf, size_t count); +ssize_t ceph_posix_pread(int fd, void *buf, size_t count, off64_t offset); +ssize_t ceph_aio_read(int fd, XrdSfsAio *aiop, AioCB *cb); +int ceph_posix_fstat(int fd, struct stat *buf); +int ceph_posix_stat(XrdOucEnv* env, const char *pathname, struct stat *buf); +int ceph_posix_fsync(int fd); +int ceph_posix_fcntl(int fd, int cmd, ... /* arg */ ); +ssize_t ceph_posix_getxattr(XrdOucEnv* env, const char* path, const char* name, + void* value, size_t size); +ssize_t ceph_posix_fgetxattr(int fd, const char* name, void* value, size_t size); +ssize_t ceph_posix_setxattr(XrdOucEnv* env, const char* path, const char* name, + const void* value, size_t size, int flags); +int ceph_posix_fsetxattr(int fd, const char* name, const void* value, size_t size, int flags); +int ceph_posix_removexattr(XrdOucEnv* env, const char* path, const char* name); +int ceph_posix_fremovexattr(int fd, const char* name); +int ceph_posix_listxattrs(XrdOucEnv* env, const char* path, XrdSysXAttr::AList **aPL, int getSz); +int ceph_posix_flistxattrs(int fd, XrdSysXAttr::AList **aPL, int getSz); +void ceph_posix_freexattrlist(XrdSysXAttr::AList *aPL); +int ceph_posix_statfs(long long *totalSpace, long long *freeSpace); +int ceph_posix_truncate(XrdOucEnv* env, const char *pathname, unsigned long long size); +int ceph_posix_ftruncate(int fd, unsigned long long size); +int ceph_posix_unlink(XrdOucEnv* env, const char *pathname); +DIR* ceph_posix_opendir(XrdOucEnv* env, const char *pathname); +int ceph_posix_readdir(DIR* dirp, char *buff, int blen); +int ceph_posix_closedir(DIR *dirp); + +#endif // __XRD_CEPH_POSIX__ diff --git a/src/XrdCeph/src/XrdCeph/XrdCephXAttr.cc b/src/XrdCeph/src/XrdCeph/XrdCephXAttr.cc new file mode 100644 index 00000000000..4c2f60a0f96 --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephXAttr.cc @@ -0,0 +1,114 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#include "XrdVersion.hh" +#include "XrdCeph/XrdCephPosix.hh" +#include "XrdSys/XrdSysError.hh" +#include "XrdOuc/XrdOucTrace.hh" +#include "XrdCeph/XrdCephXAttr.hh" + +XrdSysError XrdCephXattrEroute(0); +XrdOucTrace XrdCephXattrTrace(&XrdCephXattrEroute); + +extern "C" +{ + XrdSysXAttr* + XrdSysGetXAttrObject(XrdSysError *errP, + const char *config_fn, + const char *parms) + { + // Do the herald thing + XrdCephXattrEroute.SetPrefix("cephxattr_"); + XrdCephXattrEroute.logger(errP->logger()); + XrdCephXattrEroute.Say("++++++ CERN/IT-DSS XrdCephXattr"); + // set parameters + try { + ceph_posix_set_defaults(parms); + } catch (std::exception &e) { + XrdCephXattrEroute.Say("CephXattr loading failed with exception. Check the syntax of parameters : ", parms); + return 0; + } + return new XrdCephXAttr(); + } +} + +XrdCephXAttr::XrdCephXAttr() {} + +XrdCephXAttr::~XrdCephXAttr() {} + +int XrdCephXAttr::Del(const char *Aname, const char *Path, int fd) { + try { + return ceph_posix_removexattr(0, Path, Aname); + } catch (std::exception &e) { + XrdCephXattrEroute.Say("Del : invalid syntax in file parameters", Path); + return -EINVAL; + } +} + +void XrdCephXAttr::Free(AList *aPL) { + ceph_posix_freexattrlist(aPL); +} + +int XrdCephXAttr::Get(const char *Aname, void *Aval, int Avsz, + const char *Path, int fd) { + if (fd >= 0) { + return ceph_posix_fgetxattr(fd, Aname, Aval, Avsz); + } else { + try { + return ceph_posix_getxattr(0, Path, Aname, Aval, Avsz); + } catch (std::exception &e) { + XrdCephXattrEroute.Say("Get : invalid syntax in file parameters", Path); + return -EINVAL; + } + } +} + +int XrdCephXAttr::List(AList **aPL, const char *Path, int fd, int getSz) { + if (fd > 0) { + return ceph_posix_flistxattrs(fd, aPL, getSz); + } else { + try { + return ceph_posix_listxattrs(0, Path, aPL, getSz); + } catch (std::exception &e) { + XrdCephXattrEroute.Say("List : invalid syntax in file parameters", Path); + return -EINVAL; + } + } +} + +int XrdCephXAttr::Set(const char *Aname, const void *Aval, int Avsz, + const char *Path, int fd, int isNew) { + if (fd >= 0) { + return ceph_posix_fsetxattr(fd, Aname, Aval, Avsz, 0); + } else { + try { + return ceph_posix_setxattr(0, Path, Aname, Aval, Avsz, 0); + } catch (std::exception &e) { + XrdCephXattrEroute.Say("Set : invalid syntax in file parameters", Path); + return -EINVAL; + } + } +} + +XrdVERSIONINFO(XrdSysGetXAttrObject, XrdCephXAttr); diff --git a/src/XrdCeph/src/XrdCeph/XrdCephXAttr.hh b/src/XrdCeph/src/XrdCeph/XrdCephXAttr.hh new file mode 100644 index 00000000000..ed6f4d39f21 --- /dev/null +++ b/src/XrdCeph/src/XrdCeph/XrdCephXAttr.hh @@ -0,0 +1,152 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2014-2015 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#ifndef __XRD_CEPH_XATTR_HH__ +#define __XRD_CEPH_XATTR_HH__ + +#include + +//------------------------------------------------------------------------------ +//! This class implements XrdSysXAttr interface for usage with a CEPH storage. +//! It should be loaded via the ofs.xattrlib directive. +//! +//! This plugin is able to use any pool of ceph with any userId. +//! There are several ways to provide the pool and userId to be used for a given +//! operation. Here is the ordered list of possibilities. +//! First one defined wins : +//! - the path can be prepended with userId and pool. Syntax is : +//! [[userId@]pool:] +//! - the XrdOucEnv parameter, when existing, can have 'cephUserId' and/or +//! 'cephPool' entries +//! - the ofs.xattrlib directive can provide an argument with format : +//! [userID@]pool +//! - default are 'admin' and 'default' for userId and pool respectively +//! +//! Note that the definition of a default via the ofs.xattrlib directive may +//! clash with one used in a ofs.osslib directive. In case both directives +//! have a default and they are different, the behavior is not defined. +//! In case one of the two only has a default, it will be applied for both plugins. +//------------------------------------------------------------------------------ + +class XrdCephXAttr : public XrdSysXAttr { + +public: + + //------------------------------------------------------------------------------ + //! Constructor + //------------------------------------------------------------------------------ + XrdCephXAttr(); + + //------------------------------------------------------------------------------ + //! Destructor + //------------------------------------------------------------------------------ + virtual ~XrdCephXAttr(); + + //------------------------------------------------------------------------------ + //! Remove an extended attribute. + //! + //! @param Aname -> The attribute name. + //! @param Path -> Path of the file whose attribute is to be removed. + //! @param fd If >=0 is the file descriptor of the opened subject file. + //! + //! @return =0 Attribute was successfully removed or did not exist. + //! @return <0 Attribute was not removed, the return value is -errno that + //! describes the reason for the failure. + //------------------------------------------------------------------------------ + virtual int Del(const char *Aname, const char *Path, int fd=-1); + + //------------------------------------------------------------------------------ + //! Release storage occupied by the Alist structure returned by List(). + //! + //! @param aPL -> The first element of the AList structure. + //------------------------------------------------------------------------------ + + virtual void Free(AList *aPL); + + //------------------------------------------------------------------------------ + //! Get an attribute value and its size. + //! + //! @param Aname -> The attribute name. + //! @param Aval -> Buffer to receive the attribute value. + //! @param Avsz Length of the buffer in bytes. Only up to this number of + //! bytes should be returned. However, should Avsz be zero + //! the the size of the attribute value should be returned + //! and the Aval argument should be ignored. + //! @param Path -> Path of the file whose attribute is to be fetched. + //! @param fd -> If >=0 is the file descriptor of the opened subject file. + //! + //! @return >0 The number of bytes placed in Aval. However, if avsz is zero + //! then the value is the actual size of the attribute value. + //! @return =0 The attribute does not exist. + //! @return <0 The attribute value could not be returned. The returned + //! value is -errno describing the reason. + //------------------------------------------------------------------------------ + + virtual int Get(const char *Aname, void *Aval, int Avsz, + const char *Path, int fd=-1); + + //------------------------------------------------------------------------------ + //! Get all of the attributes associated with a file. + //! + //! @param aPL -> the pointer to hold the first element of AList. The + //! storage occupied by the returned AList must be released + //! by calling Free(). + //! @param Path -> Path of the file whose attributes are t be returned. + //! @param fd -> If >=0 is the file descriptor of the opened subject file. + //! @param getSz When != 0 then the size of the maximum attribute value + //! should be returned. Otherwise, upon success 0 is returned. + //! + //! @return >0 Attributes were returned and aPL points to the first + //! attribute value. The returned value is the largest size + //! of an attribute value encountered (getSz != 0). + //! @return =0 Attributes were returned and aPL points to the first + //! attribute value (getSz == 0). + //! @return <0 The attribute values could not be returned. The returned + //! value is -errno describing the reason. + //------------------------------------------------------------------------------ + virtual int List(AList **aPL, const char *Path, int fd=-1, int getSz=0); + + //------------------------------------------------------------------------------ + //! Set an attribute. + //! + //! @param Aname -> The attribute name. + //! @param Aval -> Buffer holding the attribute value. + //! @param Avsz Length of the buffer in bytes. This is the length of the + //! attribute value which may contain binary data. + //! @param Path -> Path of the file whose attribute is to be set. + //! @param fd -> If >=0 is the file descriptor of the opened subject file. + //! @param isNew When !0 then the attribute must not exist (i.e. new). + //! Otherwise, if it does exist, the value is replaced. In + //! either case, if it does not exist it should be created. + //! + //! @return =0 The attribute was successfully set. + //! @return <0 The attribute values could not be set. The returned + //! value is -errno describing the reason. + //------------------------------------------------------------------------------ + virtual int Set(const char *Aname, const void *Aval, int Avsz, + const char *Path, int fd=-1, int isNew=0); + +}; + +#endif /* __XRD_CEPH_XATTR_HH__ */ diff --git a/src/XrdCeph/src/XrdVersion.hh.in b/src/XrdCeph/src/XrdVersion.hh.in new file mode 100644 index 00000000000..8ccbfa4c07b --- /dev/null +++ b/src/XrdCeph/src/XrdVersion.hh.in @@ -0,0 +1,106 @@ +/******************************************************************************/ +/* */ +/* X r d V e r s i o n . h h . i n */ +/* */ +/* (c) 2012 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +// this file is automatically updated by the genversion.sh script +// if you touch anything make sure that it still works + +#ifndef __XRD_VERSION_H__ +#define __XRD_VERSION_H__ + +#define XrdVERSION "unknown" + +// Numeric representation of the version tag +// The format for the released code is: xyyzz, where: x is the major version, +// y is the minor version and zz is the bugfix revision number +// For the non-released code the value is 1000000 +#define XrdVNUMUNK 1000000 +#define XrdVNUMBER 1000000 + +#if XrdDEBUG +#define XrdVSTRING XrdVERSION "_dbg" +#else +#define XrdVSTRING XrdVERSION +#endif + +// The following defines the shared library version number of any plug-in. +// Generally, all plug-ins have a uniform version number releative to a +// specific compilation. This version is appended to the so-name and for +// dylibs becomes part of he actual filename (MacOS format). +// +#ifndef XRDPLUGIN_SOVERSION +#define XRDPLUGIN_SOVERSION "4" +#endif + +#define XrdDEFAULTPORT 1094; + +// The following macros extract version digits from a numeric version number +#define XrdMajorVNUM(x) x/10000 +#define XrdMinorVNUM(x) x/100%100 +#define XrdPatchVNUM(x) x%100 + +// The following structure defines the standard way to record a version. You can +// determine component version numbers within an object file by simply executing +// "strings | grep '@V:'". +// +struct XrdVersionInfo {int vNum; const char vOpt; const char vPfx[3];\ + const char vStr[40];}; + +// Macro to define the suffix to use when generating the extern version symbol. +// This is used by SysPlugin. We cannot use it here as cpp does not expand the +// macro when catenating tokens togther and we want to avoid yet another macro. +// +#define XrdVERSIONINFOSFX "_" + +// The following macro defines a local copy of version information. Parameters: +// x -> The variable name of the version information structure +// y -> An unquoted 1- to 15-character component name (e.g. cmsd, seckrb5) +// vn -> The integer version number to be used +// vs -> The string version number to be used +// +#define XrdVERSIONINFODEF(x,y,vn,vs) \ + XrdVersionInfo x = \ + {vn, (sizeof(#y)-1) & 0x0f,{'@','V',':'}, #y " " vs} + +// The following macro defines an externally referencable structure that records +// the version used to compile code. It is used by the plugin loader. Parms: +// x -> The variable name of the version information structure +// y -> An unquoted 1- to 15-character component name (e.g. cmsd, seckrb5, etc). +// +#define XrdVERSIONINFO(x,y) \ + extern "C" {XrdVERSIONINFODEF(x##_,y,XrdVNUMBER,XrdVERSION);} + +// The following macro is an easy way to declare externally defined version +// information. This macro must be used at file level. +// +#define XrdVERSIONINFOREF(x) extern "C" XrdVersionInfo x##_ + +// The following macro can be used to reference externally defined version +// information. As the composition of the symbolic name may change you should +// use this macro to refer to the version information declaration. +// +#define XrdVERSIONINFOVAR(x) x##_ +#endif diff --git a/src/XrdCeph/tests/CMakeLists.txt b/src/XrdCeph/tests/CMakeLists.txt new file mode 100644 index 00000000000..7216d36fd8e --- /dev/null +++ b/src/XrdCeph/tests/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory( XrdCephTests ) diff --git a/src/XrdCeph/tests/XrdCephTests/CMakeLists.txt b/src/XrdCeph/tests/XrdCephTests/CMakeLists.txt new file mode 100644 index 00000000000..2011390d85b --- /dev/null +++ b/src/XrdCeph/tests/XrdCephTests/CMakeLists.txt @@ -0,0 +1,27 @@ +message( "XROOTD_INCLUDE_DIR : ${XROOTD_INCLUDE_DIR}" ) + +add_library( + XrdCephTests MODULE + CephParsingTest.cc +) + +target_link_libraries( + XrdCephTests + pthread + ${CPPUNIT_LIBRARIES} + ${ZLIB_LIBRARY} + XrdCephPosix ) + +target_include_directories(XrdCephTests PRIVATE + ${CPPUNIT_INCLUDE_DIRS} + ${RADOS_INCLUDE_DIR} + ${XROOTD_INCLUDE_DIR} + ${PROJECT_SOURCE_DIR}/src) + +#------------------------------------------------------------------------------- +# Install +#------------------------------------------------------------------------------- +install( + TARGETS XrdCephTests + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) diff --git a/src/XrdCeph/tests/XrdCephTests/CephParsingTest.cc b/src/XrdCeph/tests/XrdCephTests/CephParsingTest.cc new file mode 100644 index 00000000000..194bd5e40c8 --- /dev/null +++ b/src/XrdCeph/tests/XrdCephTests/CephParsingTest.cc @@ -0,0 +1,149 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN) +// Author: Sebastien Ponce +//------------------------------------------------------------------------------ +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +//------------------------------------------------------------------------------ + +#include +#include +#include + +#define MB 1024*1024 +struct CephFile { + std::string name; + std::string pool; + std::string userId; + unsigned int nbStripes; + unsigned long long stripeUnit; + unsigned long long objectSize; +}; +void fillCephFileParams(const std::string ¶ms, XrdOucEnv *env, CephFile &file); +void fillCephFile(const char *path, XrdOucEnv *env, CephFile &file); + +//------------------------------------------------------------------------------ +// Declaration +//------------------------------------------------------------------------------ +class CephParsingTest: public CppUnit::TestCase +{ + public: + CPPUNIT_TEST_SUITE( CephParsingTest ); + CPPUNIT_TEST( ParamTest ); + CPPUNIT_TEST( FileTest ); + CPPUNIT_TEST_SUITE_END(); + void ParamTest(); + void FileTest(); +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( CephParsingTest ); + +//------------------------------------------------------------------------------ +// Helper functions +//------------------------------------------------------------------------------ +static CephFile parseParam(std::string param, XrdOucEnv *env= NULL) { + CephFile cf; + fillCephFileParams(param, env, cf); + return cf; +} + +static CephFile parseFile(std::string param, XrdOucEnv *env= NULL) { + CephFile cf; + fillCephFile(param.c_str(), env, cf); + return cf; +} + +void checkResult(CephFile a, CephFile b) { + std::cout << a.name << " " << a.pool << " " << a.userId + << " " << a.nbStripes << " " << a.stripeUnit << " " << a.objectSize + << " / " << b.name << " " << b.pool << " " << b.userId + << " " << b.nbStripes << " " << b.stripeUnit << " " << b.objectSize + << std::endl; + CPPUNIT_ASSERT(a.name == b.name); + CPPUNIT_ASSERT(a.pool == b.pool); + CPPUNIT_ASSERT(a.userId == b.userId); + CPPUNIT_ASSERT(a.nbStripes == b.nbStripes); + CPPUNIT_ASSERT(a.stripeUnit == b.stripeUnit); + CPPUNIT_ASSERT(a.objectSize == b.objectSize); +} + +//------------------------------------------------------------------------------ +// Param test +//------------------------------------------------------------------------------ +void CephParsingTest::ParamTest() { + std::map inputs; + inputs[""] = (CephFile){"", "default", "admin", 1, 4*MB, 4*MB}; + inputs["pool"] = (CephFile){"", "pool", "admin", 1, 4*MB, 4*MB}; + inputs["@"] = (CephFile){"", "default", "", 1, 4*MB, 4*MB}; + inputs["@pool"] = (CephFile){"", "pool", "", 1, 4*MB, 4*MB}; + inputs["user@"] = (CephFile){"", "default", "user", 1, 4*MB, 4*MB}; + inputs["user@pool"] = (CephFile){"", "pool", "user", 1, 4*MB, 4*MB}; + inputs["pool,1"] = (CephFile){"", "pool", "admin", 1, 4*MB, 4*MB}; + inputs["user@pool,1"] = (CephFile){"", "pool", "user", 1, 4*MB, 4*MB}; + inputs["pool,5"] = (CephFile){"", "pool", "admin", 5, 4*MB, 4*MB}; + inputs["user@pool,5"] = (CephFile){"", "pool", "user", 5, 4*MB, 4*MB}; + inputs["pool,5,200"] = (CephFile){"", "pool", "admin", 5, 200, 4*MB}; + inputs["user@pool,5,200"] = (CephFile){"", "pool", "user", 5, 200, 4*MB}; + inputs["pool,5,200,800"] = (CephFile){"", "pool", "admin", 5, 200, 800}; + inputs["user@pool,5,200,800"] = (CephFile){"", "pool", "user", 5, 200, 800}; + for (std::map::const_iterator it = inputs.begin(); + it != inputs.end(); + it++) { + std::cout << it->first << std::endl; + checkResult(parseParam(it->first), it->second); + } +} + +//------------------------------------------------------------------------------ +// File test +//------------------------------------------------------------------------------ +void CephParsingTest::FileTest() { + std::map inputs; + std::vector filenames; + filenames.push_back(""); + filenames.push_back("foo"); + filenames.push_back("/foo/bar"); + filenames.push_back("foo@bar"); + filenames.push_back("foo@bar,1"); + filenames.push_back("foo@bar,1,2"); + filenames.push_back("foo@bar,1,2,3"); + filenames.push_back("foo:bar"); + filenames.push_back(":foo"); + for (std::vector::const_iterator it = filenames.begin(); + it != filenames.end(); + it++) { + if (std::string::npos == it->find(':')) { + inputs[*it] = (CephFile){*it, "default", "admin", 1, 4*MB, 4*MB}; + } + inputs[":" + *it] = (CephFile){*it, "default", "admin", 1, 4*MB, 4*MB}; + inputs["pool:" + *it] = (CephFile){*it, "pool", "admin", 1, 4*MB, 4*MB}; + inputs["@:" + *it] = (CephFile){*it, "default", "", 1, 4*MB, 4*MB}; + inputs["@pool:" + *it] = (CephFile){*it, "pool", "", 1, 4*MB, 4*MB}; + inputs["user@:" + *it] = (CephFile){*it, "default", "user", 1, 4*MB, 4*MB}; + inputs["user@pool:" + *it] = (CephFile){*it, "pool", "user", 1, 4*MB, 4*MB}; + inputs["pool,1:" + *it] = (CephFile){*it, "pool", "admin", 1, 4*MB, 4*MB}; + inputs["user@pool,1:" + *it] = (CephFile){*it, "pool", "user", 1, 4*MB, 4*MB}; + inputs["pool,5:" + *it] = (CephFile){*it, "pool", "admin", 5, 4*MB, 4*MB}; + inputs["user@pool,5:" + *it] = (CephFile){*it, "pool", "user", 5, 4*MB, 4*MB}; + inputs["pool,5,200:" + *it] = (CephFile){*it, "pool", "admin", 5, 200, 4*MB}; + inputs["user@pool,5,200:" + *it] = (CephFile){*it, "pool", "user", 5, 200, 4*MB}; + inputs["pool,5,200,800:" + *it] = (CephFile){*it, "pool", "admin", 5, 200, 800}; + inputs["user@pool,5,200,800:" + *it] = (CephFile){*it, "pool", "user", 5, 200, 800}; + } + for (std::map::const_iterator it = inputs.begin(); + it != inputs.end(); + it++) { + std::cout << it->first << std::endl; + checkResult(parseFile(it->first), it->second); + } +} diff --git a/src/XrdCks/XrdCksAssist.hh b/src/XrdCks/XrdCksAssist.hh index ccd9d21a9b8..e5baabc2f6c 100644 --- a/src/XrdCks/XrdCksAssist.hh +++ b/src/XrdCks/XrdCksAssist.hh @@ -33,6 +33,8 @@ #include #include +#include + //------------------------------------------------------------------------------ //! This header file defines linkages to various XRootD checksum assistants. //! The functions described here are located in libXrdUtils.so. diff --git a/src/XrdCl/CMakeLists.txt b/src/XrdCl/CMakeLists.txt index 2179019af8c..a8fc5424271 100644 --- a/src/XrdCl/CMakeLists.txt +++ b/src/XrdCl/CMakeLists.txt @@ -1,5 +1,4 @@ -include( XRootDCommon ) #------------------------------------------------------------------------------- # Shared library version @@ -109,6 +108,7 @@ add_library( target_link_libraries( XrdCl + PRIVATE XrdXml XrdUtils uuid::uuid @@ -126,7 +126,7 @@ set_target_properties( if( BUILD_XRDEC ) target_include_directories(XrdCl PUBLIC ${ISAL_INCLUDE_DIRS}) - target_link_libraries(XrdCl ${ISAL_LIBRARIES}) + target_link_libraries(XrdCl PRIVATE ${ISAL_LIBRARIES}) endif() #------------------------------------------------------------------------------- @@ -145,6 +145,10 @@ target_link_libraries( XrdUtils ${READLINE_LIBRARY} ${NCURSES_LIBRARY} ) + +if( READLINE_FOUND ) + target_include_directories(xrdfs PRIVATE ${READLINE_INCLUDE_DIR}) +endif() endif() #------------------------------------------------------------------------------- @@ -158,6 +162,7 @@ add_executable( target_link_libraries( xrdcp XrdCl + XrdUtils XrdAppUtils ) endif() @@ -193,7 +198,6 @@ install( XrdClXRootDResponses.hh XrdClOptional.hh XrdClPlugInInterface.hh - XrdClPlugInManager.hh XrdClPropertyList.hh XrdClLog.hh DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xrootd/XrdCl ) @@ -201,11 +205,14 @@ install( install( FILES # Additional client headers + XrdClJobManager.hh XrdClMessage.hh + XrdClPlugInManager.hh XrdClPostMaster.hh XrdClPostMasterInterfaces.hh XrdClTransportManager.hh XrdClResponseJob.hh + XrdClSyncQueue.hh XrdClZipArchive.hh XrdClZipCache.hh # Declarative operations @@ -219,17 +226,12 @@ install( XrdClFileOperations.hh XrdClFileSystemOperations.hh XrdClFinalOperation.hh + XrdClUtils.hh + XrdClXRootDTransport.hh XrdClZipOperations.hh DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xrootd/private/XrdCl ) if( NOT XRDCL_LIB_ONLY ) -install( - FILES - ${PROJECT_SOURCE_DIR}/docs/man/xrdfs.1 - ${PROJECT_SOURCE_DIR}/docs/man/xrdcp.1 - ${PROJECT_SOURCE_DIR}/docs/man/xrdmapc.1 - DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 ) - install( CODE " EXECUTE_PROCESS( @@ -237,26 +239,4 @@ install( WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR} )" ) -install( - CODE " - EXECUTE_PROCESS( - COMMAND ln -sf xrdcp.1 xrdcopy.1 - WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man1 )" -) - -install( - CODE " - FOREACH(MANPAGE xrdfs.1 xrdcp.1 xrdmapc.1) - MESSAGE( \"-- Processing: \" \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man1/\${MANPAGE} ) - EXECUTE_PROCESS( - COMMAND cat \${MANPAGE} - COMMAND sed -e \"s/__VERSION__/${XROOTD_VERSION}/\" - OUTPUT_FILE \${MANPAGE}.new - WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man1 ) - EXECUTE_PROCESS( - COMMAND mv -f \${MANPAGE}.new \${MANPAGE} - WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man1 ) - ENDFOREACH()" -) - endif() diff --git a/src/XrdCl/XrdClArg.hh b/src/XrdCl/XrdClArg.hh index 3ff9fce9f5b..147d73b51cb 100644 --- a/src/XrdCl/XrdClArg.hh +++ b/src/XrdCl/XrdClArg.hh @@ -29,6 +29,7 @@ #include "XrdCl/XrdClFwd.hh" #include "XrdCl/XrdClOptional.hh" +#include #include #include #include diff --git a/src/XrdCl/XrdClAsyncSocketHandler.cc b/src/XrdCl/XrdClAsyncSocketHandler.cc index c416cebb367..1d380a8ecb7 100644 --- a/src/XrdCl/XrdClAsyncSocketHandler.cc +++ b/src/XrdCl/XrdClAsyncSocketHandler.cc @@ -136,6 +136,9 @@ namespace XrdCl } pHandShakeDone = false; + pTlsHandShakeOngoing = false; + pHSWaitStarted = 0; + pHSWaitSeconds = 0; //-------------------------------------------------------------------------- // Initiate async connection to the address @@ -213,6 +216,23 @@ namespace XrdCl //-------------------------------------------------------------------------- type = pSocket->MapEvent( type ); + //-------------------------------------------------------------------------- + // Handle any read or write events. If any of the handlers indicate an error + // we will have been disconnected. A disconnection may cause the current + // object to be asynchronously reused or deleted, so we return immediately. + //-------------------------------------------------------------------------- + if( !EventRead( type ) ) + return; + + if( !EventWrite( type ) ) + return; + } + + //---------------------------------------------------------------------------- + // Handler for read related socket events + //---------------------------------------------------------------------------- + bool AsyncSocketHandler::EventRead( uint8_t type ) + { //-------------------------------------------------------------------------- // Read event //-------------------------------------------------------------------------- @@ -220,11 +240,12 @@ namespace XrdCl { pLastActivity = time(0); if( unlikely( pTlsHandShakeOngoing ) ) - OnTLSHandShake(); - else if( likely( pHandShakeDone ) ) - OnRead(); - else - OnReadWhileHandshaking(); + return OnTLSHandShake(); + + if( likely( pHandShakeDone ) ) + return OnRead(); + + return OnReadWhileHandshaking(); } //-------------------------------------------------------------------------- @@ -233,14 +254,25 @@ namespace XrdCl else if( type & ReadTimeOut ) { if( pHSWaitSeconds ) - CheckHSWait(); + { + if( !CheckHSWait() ) + return false; + } if( likely( pHandShakeDone ) ) - OnReadTimeout(); - else - OnTimeoutWhileHandshaking(); + return OnReadTimeout(); + + return OnTimeoutWhileHandshaking(); } + return true; + } + + //---------------------------------------------------------------------------- + // Handler for write related socket events + //---------------------------------------------------------------------------- + bool AsyncSocketHandler::EventWrite( uint8_t type ) + { //-------------------------------------------------------------------------- // Write event //-------------------------------------------------------------------------- @@ -248,19 +280,21 @@ namespace XrdCl { pLastActivity = time(0); if( unlikely( pSocket->GetStatus() == Socket::Connecting ) ) - OnConnectionReturn(); + return OnConnectionReturn(); + //------------------------------------------------------------------------ // Make sure we are not writing anything if we have been told to wait. //------------------------------------------------------------------------ - else if( pHSWaitSeconds == 0 ) - { - if( unlikely( pTlsHandShakeOngoing ) ) - OnTLSHandShake(); - else if( likely( pHandShakeDone ) ) - OnWrite(); - else - OnWriteWhileHandshaking(); - } + if( pHSWaitSeconds != 0 ) + return true; + + if( unlikely( pTlsHandShakeOngoing ) ) + return OnTLSHandShake(); + + if( likely( pHandShakeDone ) ) + return OnWrite(); + + return OnWriteWhileHandshaking(); } //-------------------------------------------------------------------------- @@ -269,16 +303,18 @@ namespace XrdCl else if( type & WriteTimeOut ) { if( likely( pHandShakeDone ) ) - OnWriteTimeout(); - else - OnTimeoutWhileHandshaking(); + return OnWriteTimeout(); + + return OnTimeoutWhileHandshaking(); } + + return true; } //---------------------------------------------------------------------------- // Connect returned //---------------------------------------------------------------------------- - void AsyncSocketHandler::OnConnectionReturn() + bool AsyncSocketHandler::OnConnectionReturn() { //-------------------------------------------------------------------------- // Check whether we were able to connect @@ -303,7 +339,7 @@ namespace XrdCl XrdSysE2T( errno ) ); pStream->OnConnectError( pSubStreamNum, XRootDStatus( stFatal, errSocketOptError, errno ) ); - return; + return false; } //-------------------------------------------------------------------------- @@ -315,7 +351,7 @@ namespace XrdCl pStreamName.c_str(), XrdSysE2T( errorCode ) ); pStream->OnConnectError( pSubStreamNum, XRootDStatus( stError, errConnectionError ) ); - return; + return false; } pSocket->SetStatus( Socket::Connected ); @@ -326,7 +362,7 @@ namespace XrdCl if( !st.IsOK() ) { pStream->OnConnectError( pSubStreamNum, st ); - return; + return false; } //-------------------------------------------------------------------------- @@ -344,7 +380,7 @@ namespace XrdCl log->Error( AsyncSockMsg, "[%s] Connection negotiation failed", pStreamName.c_str() ); pStream->OnConnectError( pSubStreamNum, st ); - return; + return false; } if( st.code != suRetry ) @@ -372,19 +408,20 @@ namespace XrdCl { pStream->OnConnectError( pSubStreamNum, XRootDStatus( stFatal, errPollerError ) ); - return; + return false; } + return true; } //---------------------------------------------------------------------------- // Got a write readiness event //---------------------------------------------------------------------------- - void AsyncSocketHandler::OnWrite() + bool AsyncSocketHandler::OnWrite() { if( !reqwriter ) { OnFault( XRootDStatus( stError, errInternal, 0, "Request writer is null." ) ); - return; + return false; } //-------------------------------------------------------------------------- // Let's do the writing ... @@ -396,30 +433,34 @@ namespace XrdCl // We failed //------------------------------------------------------------------------ OnFault( st ); - return; + return false; } //-------------------------------------------------------------------------- // We are not done yet //-------------------------------------------------------------------------- - if( st.code == suRetry) return; + if( st.code == suRetry) return true; //-------------------------------------------------------------------------- // Disable the respective substream if empty //-------------------------------------------------------------------------- reqwriter->Reset(); pStream->DisableIfEmpty( pSubStreamNum ); + return true; } //---------------------------------------------------------------------------- // Got a write readiness event while handshaking //---------------------------------------------------------------------------- - void AsyncSocketHandler::OnWriteWhileHandshaking() + bool AsyncSocketHandler::OnWriteWhileHandshaking() { XRootDStatus st; if( !hswriter || !hswriter->HasMsg() ) { if( !(st = DisableUplink()).IsOK() ) + { OnFaultWhileHandshaking( st ); - return; + return false; + } + return true; } //-------------------------------------------------------------------------- // Let's do the writing ... @@ -431,25 +472,29 @@ namespace XrdCl // We failed //------------------------------------------------------------------------ OnFaultWhileHandshaking( st ); - return; + return false; } //-------------------------------------------------------------------------- // We are not done yet //-------------------------------------------------------------------------- - if( st.code == suRetry ) return; + if( st.code == suRetry ) return true; //-------------------------------------------------------------------------- // Disable the uplink // Note: at this point we don't deallocate the HS message as we might need // to re-send it in case of a kXR_wait response //-------------------------------------------------------------------------- if( !(st = DisableUplink()).IsOK() ) + { OnFaultWhileHandshaking( st ); + return false; + } + return true; } //---------------------------------------------------------------------------- // Got a read readiness event //---------------------------------------------------------------------------- - void AsyncSocketHandler::OnRead() + bool AsyncSocketHandler::OnRead() { //-------------------------------------------------------------------------- // Make sure the response reader object exists @@ -457,7 +502,7 @@ namespace XrdCl if( !rspreader ) { OnFault( XRootDStatus( stError, errInternal, 0, "Response reader is null." ) ); - return; + return false; } //-------------------------------------------------------------------------- @@ -471,7 +516,7 @@ namespace XrdCl if( !st.IsOK() && st.code == errCorruptedHeader ) { OnHeaderCorruption(); - return; + return false; } //-------------------------------------------------------------------------- @@ -480,24 +525,25 @@ namespace XrdCl if( !st.IsOK() ) { OnFault( st ); - return; + return false; } //-------------------------------------------------------------------------- // We are not done yet //-------------------------------------------------------------------------- - if( st.code == suRetry ) return; + if( st.code == suRetry ) return true; //-------------------------------------------------------------------------- // We are done, reset the response reader so we can read out next message //-------------------------------------------------------------------------- rspreader->Reset(); + return true; } //---------------------------------------------------------------------------- // Got a read readiness event while handshaking //---------------------------------------------------------------------------- - void AsyncSocketHandler::OnReadWhileHandshaking() + bool AsyncSocketHandler::OnReadWhileHandshaking() { //-------------------------------------------------------------------------- // Make sure the response reader object exists @@ -505,7 +551,7 @@ namespace XrdCl if( !hsreader ) { OnFault( XRootDStatus( stError, errInternal, 0, "Hand-shake reader is null." ) ); - return; + return false; } //-------------------------------------------------------------------------- @@ -516,19 +562,19 @@ namespace XrdCl if( !st.IsOK() ) { OnFaultWhileHandshaking( st ); - return; + return false; } if( st.code != suDone ) - return; + return true; - HandleHandShake( hsreader->ReleaseMsg() ); + return HandleHandShake( hsreader->ReleaseMsg() ); } //------------------------------------------------------------------------ // Handle the handshake message //------------------------------------------------------------------------ - void AsyncSocketHandler::HandleHandShake( std::unique_ptr msg ) + bool AsyncSocketHandler::HandleHandShake( std::unique_ptr msg ) { //-------------------------------------------------------------------------- // OK, we have a new message, let's deal with it; @@ -547,7 +593,7 @@ namespace XrdCl if( !st.IsOK() ) { OnFaultWhileHandshaking( st ); - return; + return false; } if( st.code == suRetry ) @@ -568,6 +614,7 @@ namespace XrdCl pStreamName.c_str() ); OnFaultWhileHandshaking( XRootDStatus( stError, errSocketTimeout ) ); + return false; } else { @@ -577,19 +624,18 @@ namespace XrdCl Log *log = DefaultEnv::GetLog(); log->Debug( AsyncSockMsg, "[%s] Received a wait response to endsess request, " "will wait for %d seconds before replaying the endsess request", - waitSeconds ); + pStreamName.c_str(), waitSeconds ); pHSWaitStarted = time( 0 ); pHSWaitSeconds = waitSeconds; } - return; + return true; } //------------------------------------------------------------------------ // We are re-sending a protocol request //------------------------------------------------------------------------ else if( pHandShakeData->out ) { - SendHSMsg(); - return; + return SendHSMsg(); } } @@ -600,19 +646,22 @@ namespace XrdCl pTransport->NeedEncryption( pHandShakeData.get(), *pChannelData ) ) { XRootDStatus st = DoTlsHandShake(); - if( !st.IsOK() || st.code == suRetry ) return; + if( !st.IsOK() ) + return false; + if ( st.code == suRetry ) + return true; } //-------------------------------------------------------------------------- // Now prepare the next step of the hand-shake procedure //-------------------------------------------------------------------------- - HandShakeNextStep( st.IsOK() && st.code == suDone ); + return HandShakeNextStep( st.IsOK() && st.code == suDone ); } //------------------------------------------------------------------------ // Prepare the next step of the hand-shake procedure //------------------------------------------------------------------------ - void AsyncSocketHandler::HandShakeNextStep( bool done ) + bool AsyncSocketHandler::HandShakeNextStep( bool done ) { //-------------------------------------------------------------------------- // We successfully proceeded to the next step @@ -636,7 +685,7 @@ namespace XrdCl if( !(st = EnableUplink()).IsOK() ) { OnFaultWhileHandshaking( st ); - return; + return false; } pHandShakeDone = true; pStream->OnConnect( pSubStreamNum ); @@ -646,8 +695,9 @@ namespace XrdCl //-------------------------------------------------------------------------- else if( pHandShakeData->out ) { - SendHSMsg(); + return SendHSMsg(); } + return true; } //---------------------------------------------------------------------------- @@ -677,27 +727,31 @@ namespace XrdCl //---------------------------------------------------------------------------- // Handle write timeout //---------------------------------------------------------------------------- - void AsyncSocketHandler::OnWriteTimeout() + bool AsyncSocketHandler::OnWriteTimeout() { - pStream->OnWriteTimeout( pSubStreamNum ); + return pStream->OnWriteTimeout( pSubStreamNum ); } //---------------------------------------------------------------------------- // Handler read timeout //---------------------------------------------------------------------------- - void AsyncSocketHandler::OnReadTimeout() + bool AsyncSocketHandler::OnReadTimeout() { - pStream->OnReadTimeout( pSubStreamNum ); + return pStream->OnReadTimeout( pSubStreamNum ); } //---------------------------------------------------------------------------- // Handle timeout while handshaking //---------------------------------------------------------------------------- - void AsyncSocketHandler::OnTimeoutWhileHandshaking() + bool AsyncSocketHandler::OnTimeoutWhileHandshaking() { time_t now = time(0); if( now > pConnectionStarted+pConnectionTimeout ) + { OnFaultWhileHandshaking( XRootDStatus( stError, errSocketTimeout ) ); + return false; + } + return true; } //---------------------------------------------------------------------------- @@ -723,8 +777,8 @@ namespace XrdCl XRootDStatus st; if( !( st = pSocket->TlsHandShake( this, pUrl.GetHostName() ) ).IsOK() ) { - OnFaultWhileHandshaking( st ); pTlsHandShakeOngoing = false; + OnFaultWhileHandshaking( st ); return st; } @@ -743,25 +797,28 @@ namespace XrdCl //---------------------------------------------------------------------------- // Handle read/write event if we are in the middle of a TLS hand-shake //---------------------------------------------------------------------------- - inline void AsyncSocketHandler::OnTLSHandShake() + inline bool AsyncSocketHandler::OnTLSHandShake() { XRootDStatus st = DoTlsHandShake(); - if( !st.IsOK() || st.code == suRetry ) return; + if( !st.IsOK() ) + return false; + if ( st.code == suRetry ) + return true; - HandShakeNextStep( pTransport->HandShakeDone( pHandShakeData.get(), - *pChannelData ) ); + return HandShakeNextStep( pTransport->HandShakeDone( pHandShakeData.get(), + *pChannelData ) ); } //---------------------------------------------------------------------------- // Prepare a HS writer for sending and enable uplink //---------------------------------------------------------------------------- - void AsyncSocketHandler::SendHSMsg() + bool AsyncSocketHandler::SendHSMsg() { if( !hswriter ) { OnFaultWhileHandshaking( XRootDStatus( stError, errInternal, 0, "HS writer object missing!" ) ); - return; + return false; } //-------------------------------------------------------------------------- // We only set a new HS message if this is not a replay due to kXR_wait @@ -783,8 +840,9 @@ namespace XrdCl if( !(st = EnableUplink()).IsOK() ) { OnFaultWhileHandshaking( st ); - return; + return false; } + return true; } kXR_int32 AsyncSocketHandler::HandleWaitRsp( Message *msg ) @@ -801,7 +859,7 @@ namespace XrdCl //---------------------------------------------------------------------------- // Check if HS wait time elapsed //---------------------------------------------------------------------------- - void AsyncSocketHandler::CheckHSWait() + bool AsyncSocketHandler::CheckHSWait() { time_t now = time( 0 ); if( now - pHSWaitStarted >= pHSWaitSeconds ) @@ -809,13 +867,15 @@ namespace XrdCl Log *log = DefaultEnv::GetLog(); log->Debug( AsyncSockMsg, "[%s] The hand-shake wait time elapsed, will " "replay the endsess request.", pStreamName.c_str() ); - SendHSMsg(); + if( !SendHSMsg() ) + return false; //------------------------------------------------------------------------ // Make sure the wait state is reset //------------------------------------------------------------------------ pHSWaitSeconds = 0; pHSWaitStarted = 0; } + return true; } //------------------------------------------------------------------------ diff --git a/src/XrdCl/XrdClAsyncSocketHandler.hh b/src/XrdCl/XrdClAsyncSocketHandler.hh index f696e754af9..8589715a846 100644 --- a/src/XrdCl/XrdClAsyncSocketHandler.hh +++ b/src/XrdCl/XrdClAsyncSocketHandler.hh @@ -30,6 +30,7 @@ #include "XrdCl/XrdClAsyncHSReader.hh" #include "XrdCl/XrdClAsyncMsgWriter.hh" #include "XrdCl/XrdClAsyncHSWriter.hh" +#include "XrdOuc/XrdOucCompiler.hh" namespace XrdCl { @@ -149,37 +150,38 @@ namespace XrdCl //------------------------------------------------------------------------ // Connect returned //------------------------------------------------------------------------ - virtual void OnConnectionReturn(); + virtual bool OnConnectionReturn() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Got a write readiness event //------------------------------------------------------------------------ - void OnWrite(); + bool OnWrite() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Got a write readiness event while handshaking //------------------------------------------------------------------------ - void OnWriteWhileHandshaking(); + bool OnWriteWhileHandshaking() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Got a read readiness event //------------------------------------------------------------------------ - void OnRead(); + bool OnRead() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Got a read readiness event while handshaking //------------------------------------------------------------------------ - void OnReadWhileHandshaking(); + bool OnReadWhileHandshaking() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle the handshake message //------------------------------------------------------------------------ - void HandleHandShake( std::unique_ptr msg ); + bool HandleHandShake( std::unique_ptr msg ) + XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Prepare the next step of the hand-shake procedure //------------------------------------------------------------------------ - void HandShakeNextStep( bool done ); + bool HandShakeNextStep( bool done ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle fault @@ -194,17 +196,17 @@ namespace XrdCl //------------------------------------------------------------------------ // Handle write timeout event //------------------------------------------------------------------------ - void OnWriteTimeout(); + bool OnWriteTimeout() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle read timeout event //------------------------------------------------------------------------ - void OnReadTimeout(); + bool OnReadTimeout() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle timeout event while handshaking //------------------------------------------------------------------------ - void OnTimeoutWhileHandshaking(); + bool OnTimeoutWhileHandshaking() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Handle header corruption in case of kXR_status response @@ -229,12 +231,12 @@ namespace XrdCl // Handle read/write event if we are in the middle of a TLS hand-shake //------------------------------------------------------------------------ // Handle read/write event if we are in the middle of a TLS hand-shake - void OnTLSHandShake(); + bool OnTLSHandShake() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Prepare a HS writer for sending and enable uplink //------------------------------------------------------------------------ - void SendHSMsg(); + bool SendHSMsg() XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Extract the value of a wait response @@ -248,7 +250,17 @@ namespace XrdCl //------------------------------------------------------------------------ // Check if HS wait time elapsed //------------------------------------------------------------------------ - void CheckHSWait(); + bool CheckHSWait() XRD_WARN_UNUSED_RESULT; + + //------------------------------------------------------------------------ + // Handler for read related socket events + //------------------------------------------------------------------------ + inline bool EventRead( uint8_t type ) XRD_WARN_UNUSED_RESULT; + + //------------------------------------------------------------------------ + // Handler for write related socket events + //------------------------------------------------------------------------ + inline bool EventWrite( uint8_t type ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ // Data members diff --git a/src/XrdCl/XrdClChannel.cc b/src/XrdCl/XrdClChannel.cc index 5f6b0efa3b9..c968040cfe4 100644 --- a/src/XrdCl/XrdClChannel.cc +++ b/src/XrdCl/XrdClChannel.cc @@ -132,7 +132,6 @@ namespace XrdCl Channel::~Channel() { pTickGenerator->Invalidate(); - pTaskManager->UnregisterTask( pTickGenerator ); delete pStream; pTransport->FinalizeChannel( pChannelData ); } diff --git a/src/XrdCl/XrdClConstants.hh b/src/XrdCl/XrdClConstants.hh index d553168f48e..11a1eca30b6 100644 --- a/src/XrdCl/XrdClConstants.hh +++ b/src/XrdCl/XrdClConstants.hh @@ -70,7 +70,7 @@ namespace XrdCl const int DefaultTCPKeepAliveInterval = 75; const int DefaultTCPKeepAliveProbes = 9; const int DefaultMultiProtocol = 0; - const int DefaultParallelEvtLoop = 1; + const int DefaultParallelEvtLoop = 10; const int DefaultMetalinkProcessing = 1; const int DefaultLocalMetalinkFile = 0; const int DefaultXRateThreshold = 0; diff --git a/src/XrdCl/XrdClCopy.cc b/src/XrdCl/XrdClCopy.cc index 2506da5e963..d0acd37e1ef 100644 --- a/src/XrdCl/XrdClCopy.cc +++ b/src/XrdCl/XrdClCopy.cc @@ -952,6 +952,7 @@ int main( int argc, char **argv ) return st.GetShellCode(); } CleanUpResults( resultVect ); + XrdCl::DefaultEnv::GetPostMaster()->Stop(); return 0; } diff --git a/src/XrdCl/XrdClCtx.hh b/src/XrdCl/XrdClCtx.hh index 41a167348bc..3bd4eebf7ba 100644 --- a/src/XrdCl/XrdClCtx.hh +++ b/src/XrdCl/XrdClCtx.hh @@ -27,6 +27,7 @@ #define SRC_XRDCL_XRDCLCTX_HH_ #include +#include namespace XrdCl { diff --git a/src/XrdCl/XrdClEcHandler.hh b/src/XrdCl/XrdClEcHandler.hh index e374d78ff45..4839c5378b4 100644 --- a/src/XrdCl/XrdClEcHandler.hh +++ b/src/XrdCl/XrdClEcHandler.hh @@ -32,11 +32,11 @@ namespace XrdCl std::string address; uint64_t freeSpace; FreeSpace() {}; - bool operator<(const FreeSpace &a) + bool operator<(const FreeSpace &a) const { return ((freeSpace > a.freeSpace) ? true : false); } - void Dump() + void Dump() const { std::cout << address << " : " << freeSpace << std::endl; } diff --git a/src/XrdCl/XrdClFS.cc b/src/XrdCl/XrdClFS.cc index a3660d078ea..e455d6ac1b4 100644 --- a/src/XrdCl/XrdClFS.cc +++ b/src/XrdCl/XrdClFS.cc @@ -158,6 +158,68 @@ XRootDStatus ConvertMode( Access::Mode &mode, const std::string &modeStr ) return XRootDStatus(); } +//------------------------------------------------------------------------------ +// Perform a cache operation +//------------------------------------------------------------------------------ +XRootDStatus DoCache( FileSystem *fs, + Env *env, + const FSExecutor::CommandParams &args ) +{ + //---------------------------------------------------------------------------- + // Check up the args + //---------------------------------------------------------------------------- + Log *log = DefaultEnv::GetLog(); + uint32_t argc = args.size(); + + if( argc != 3 ) + { + log->Error( AppMsg, "Wrong number of arguments." ); + return XRootDStatus( stError, errInvalidArgs, 0, + "Wrong number of arguments." ); + } + + if( args[1] != "evict" && args[1] != "fevict") + { + log->Error( AppMsg, "Invalid cache operation." ); + return XRootDStatus( stError, errInvalidArgs, 0, "Invalid cache operation." ); + } + + std::string fullPath; + if( !BuildPath( fullPath, env, args[2] ).IsOK() ) + { + log->Error( AppMsg, "Invalid cache path." ); + return XRootDStatus( stError, errInvalidArgs, 0, "Invalid cache path." ); + } + + //---------------------------------------------------------------------------- + // Create the command + //---------------------------------------------------------------------------- + std::string cmd = args[1]; + cmd.append(" "); + cmd.append(fullPath); + + //---------------------------------------------------------------------------- + // Run the operation + //---------------------------------------------------------------------------- + Buffer *response = 0; + XRootDStatus st = fs->SendCache( cmd, response ); + if( !st.IsOK() ) + { + log->Error( AppMsg, "Unable set cache %s: %s", + fullPath.c_str(), + st.ToStr().c_str() ); + return st; + } + + if( response ) + { + std::cout << response->ToString() << '\n'; + } + + delete response; + + return XRootDStatus(); +} //------------------------------------------------------------------------------ // Change current working directory //------------------------------------------------------------------------------ @@ -639,9 +701,12 @@ XRootDStatus DoRm( FileSystem *fs, } //---------------------------------------------------------------------------- - // Run the query + // Run the query: + // Parallel() will take the vector of Pipeline by reference and empty the + // vector, so rms.size() will change after the call. //---------------------------------------------------------------------------- - XRootDStatus st = WaitFor( Parallel( rms ).AtLeast( rms.size() ) ); + const size_t rs = rms.size(); + XRootDStatus st = WaitFor( Parallel( rms ).AtLeast( rs ) ); if( !st.IsOK() ) return st; @@ -1875,6 +1940,11 @@ XRootDStatus PrintHelp( FileSystem *, Env *, printf( " help\n" ); printf( " This help screen.\n\n" ); + printf( " cache {evict | fevict} \n" ); + printf( " Evict a file from a cache if not in use; while fevict\n" ); + printf( " focibly evicts the file causing any current uses of the\n" ); + printf( " file to get read failures on a subsequent read\n\n" ); + printf( " cd \n" ); printf( " Change the current working directory\n\n" ); @@ -2008,6 +2078,7 @@ FSExecutor *CreateExecutor( const URL &url ) Env *env = new Env(); env->PutString( "CWD", "/" ); FSExecutor *executor = new FSExecutor( url, env ); + executor->AddCommand( "cache", DoCache ); executor->AddCommand( "cd", DoCD ); executor->AddCommand( "chmod", DoChMod ); executor->AddCommand( "ls", DoLS ); diff --git a/src/XrdCl/XrdClFileStateHandler.cc b/src/XrdCl/XrdClFileStateHandler.cc index 90100056ed1..d9491c85c3c 100644 --- a/src/XrdCl/XrdClFileStateHandler.cc +++ b/src/XrdCl/XrdClFileStateHandler.cc @@ -2880,7 +2880,8 @@ namespace XrdCl bool FileStateHandler::IsRecoverable( const XRootDStatus &status ) const { if( status.code == errSocketError || status.code == errInvalidSession || - status.code == errTlsError || status.code == errSocketTimeout ) + status.code == errTlsError || status.code == errSocketTimeout || + status.code == errOperationInterrupted) { if( IsReadOnly() && !pDoRecoverRead ) return false; diff --git a/src/XrdCl/XrdClFileSystem.cc b/src/XrdCl/XrdClFileSystem.cc index 2d8002cdd3e..fe94656961a 100644 --- a/src/XrdCl/XrdClFileSystem.cc +++ b/src/XrdCl/XrdClFileSystem.cc @@ -1869,6 +1869,35 @@ namespace XrdCl return XRootDStatus(); } + //---------------------------------------------------------------------------- + // Send cache info to the server - async + //---------------------------------------------------------------------------- + XRootDStatus FileSystem::SendCache( const std::string &info, + ResponseHandler *handler, + uint16_t timeout ) + { + // Note: adding SendCache() to the FileSystemPlugin class breaks ABI! + // So, the class is missing this until we do a major release. TODO + //if( pPlugIn ) + // return pPlugIn->SendCache( info, handler, timeout ); + return SendSet("cache ", info, handler, timeout ); + } + + //---------------------------------------------------------------------------- + //! Send cache info to the server - sync + //---------------------------------------------------------------------------- + XRootDStatus FileSystem::SendCache( const std::string &info, + Buffer *&response, + uint16_t timeout ) + { + SyncResponseHandler handler; + Status st = SendCache( info, &handler, timeout ); + if( !st.IsOK() ) + return st; + + return MessageUtils::WaitForResponse( &handler, response ); + } + //---------------------------------------------------------------------------- // Send info to the server - async //---------------------------------------------------------------------------- @@ -1878,22 +1907,7 @@ namespace XrdCl { if( pPlugIn ) return pPlugIn->SendInfo( info, handler, timeout ); - - Message *msg; - ClientSetRequest *req; - const char *prefix = "monitor info "; - size_t prefixLen = strlen( prefix ); - MessageUtils::CreateRequest( msg, req, info.length()+prefixLen ); - - req->requestid = kXR_set; - req->dlen = info.length()+prefixLen; - msg->Append( prefix, prefixLen, 24 ); - msg->Append( info.c_str(), info.length(), 24+prefixLen ); - MessageSendParams params; params.timeout = timeout; - MessageUtils::ProcessSendParams( params ); - XRootDTransport::SetDescription( msg ); - - return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); + return SendSet("monitor info ", info, handler, timeout ); } //---------------------------------------------------------------------------- @@ -1911,6 +1925,31 @@ namespace XrdCl return MessageUtils::WaitForResponse( &handler, response ); } + //---------------------------------------------------------------------------- + // Send set request to the server - async + //---------------------------------------------------------------------------- + XRootDStatus FileSystem::SendSet( const char *prefix, + const std::string &info, + ResponseHandler *handler, + uint16_t timeout ) + { + + Message *msg; + ClientSetRequest *req; + size_t prefixLen = strlen( prefix ); + MessageUtils::CreateRequest( msg, req, info.length()+prefixLen ); + + req->requestid = kXR_set; + req->dlen = info.length()+prefixLen; + msg->Append( prefix, prefixLen, 24 ); + msg->Append( info.c_str(), info.length(), 24+prefixLen ); + MessageSendParams params; params.timeout = timeout; + MessageUtils::ProcessSendParams( params ); + XRootDTransport::SetDescription( msg ); + + return FileSystemData::Send( pImpl->fsdata, msg, handler, params ); + } + //---------------------------------------------------------------------------- // Prepare one or more files for access - async //---------------------------------------------------------------------------- diff --git a/src/XrdCl/XrdClFileSystem.hh b/src/XrdCl/XrdClFileSystem.hh index dabdaab0932..8229a234c10 100644 --- a/src/XrdCl/XrdClFileSystem.hh +++ b/src/XrdCl/XrdClFileSystem.hh @@ -640,6 +640,36 @@ namespace XrdCl uint16_t timeout = 0 ) XRD_WARN_UNUSED_RESULT; + //------------------------------------------------------------------------ + //! Send cache into the server - async + //! + //! @param info the info string to be sent + //! @param handler handler to be notified when the response arrives, + //! the response parameter will hold a Buffer object + //! if the procedure is successful + //! @param timeout timeout value, if 0 the environment default will + //! be used + //! @return status of the operation + //------------------------------------------------------------------------ + XRootDStatus SendCache( const std::string &info, + ResponseHandler *handler, + uint16_t timeout = 0 ) + XRD_WARN_UNUSED_RESULT; + + //------------------------------------------------------------------------ + //! Send cache into the server - sync + //! + //! @param info the info string to be sent + //! @param response the response (to be deleted by the user) + //! @param timeout timeout value, if 0 the environment default will + //! be used + //! @return status of the operation + //------------------------------------------------------------------------ + XRootDStatus SendCache( const std::string &info, + Buffer *&response, + uint16_t timeout = 0 ) + XRD_WARN_UNUSED_RESULT; + //------------------------------------------------------------------------ //! Send info to the server (up to 1024 characters)- async //! @@ -861,6 +891,20 @@ namespace XrdCl //------------------------------------------------------------------------ void UnLock(); + //------------------------------------------------------------------------ + //! Generic implementation of SendCache and SendInfo + //! + //! @param info : the info string to be sent + //! @param handler : handler to be notified when the response arrives. + //! @param timeout : timeout value or 0 for default. + //! @return status of the operation + //------------------------------------------------------------------------ + XRootDStatus SendSet( const char *prefix, + const std::string &info, + ResponseHandler *handler, + uint16_t timeout = 0 ) + XRD_WARN_UNUSED_RESULT; + //------------------------------------------------------------------------ //! Generic implementation of xattr operation //! diff --git a/src/XrdCl/XrdClFinalOperation.hh b/src/XrdCl/XrdClFinalOperation.hh index 2204c04869b..19d6102ee19 100644 --- a/src/XrdCl/XrdClFinalOperation.hh +++ b/src/XrdCl/XrdClFinalOperation.hh @@ -30,6 +30,8 @@ namespace XrdCl { + class XRootDStatus; + //--------------------------------------------------------------------------- //! Final operation in the pipeline, always executed, no matter if the //! pipeline failed or not. diff --git a/src/XrdCl/XrdClLocalFileHandler.cc b/src/XrdCl/XrdClLocalFileHandler.cc index e25d8f2a2ba..42cce9a30c2 100644 --- a/src/XrdCl/XrdClLocalFileHandler.cc +++ b/src/XrdCl/XrdClLocalFileHandler.cc @@ -664,6 +664,12 @@ namespace XrdCl std::unique_ptr buffer; int size = xattr->Get( name.c_str(), 0, 0, 0, fd ); + if( size < 0 ) + { + XRootDStatus status( stError, errLocalError, -size ); + response.push_back( XAttr( *itr, "", status ) ); + continue; + } buffer.reset( new char[size] ); int ret = xattr->Get( name.c_str(), buffer.get(), size, 0, fd ); diff --git a/src/XrdCl/XrdClOperations.cc b/src/XrdCl/XrdClOperations.cc index 611c72df727..6bba0abd0b9 100644 --- a/src/XrdCl/XrdClOperations.cc +++ b/src/XrdCl/XrdClOperations.cc @@ -210,6 +210,32 @@ namespace XrdCl final = std::move( f ); } + //------------------------------------------------------------------------ + // Called by a pipeline on the handler of its first operation before Run + //------------------------------------------------------------------------ + void PipelineHandler::PreparePipelineStart() + { + // Move any final-function from the handler of the last operaiton to the + // first. It will be moved along the pipeline of handlers while the + // pipeline is run. + + if( final || !nextOperation ) return; + PipelineHandler *last = nextOperation->handler.get(); + while( last ) + { + Operation *nextop = last->nextOperation.get(); + if( !nextop ) break; + last = nextop->handler.get(); + } + if( last ) + { + // swap-then-move rather than only move as we need to guarantee that + // last->final is left without target. + std::function f; + f.swap( last->final ); + Assign( std::move( f ) ); + } + } //------------------------------------------------------------------------ // Stop the current pipeline diff --git a/src/XrdCl/XrdClOperations.hh b/src/XrdCl/XrdClOperations.hh index 24aa66a09c7..e49bb4e84cf 100644 --- a/src/XrdCl/XrdClOperations.hh +++ b/src/XrdCl/XrdClOperations.hh @@ -123,6 +123,11 @@ namespace XrdCl //------------------------------------------------------------------------ void Assign( std::function final ); + //------------------------------------------------------------------------ + //! Called by a pipeline on the handler of its first operation before Run + //------------------------------------------------------------------------ + void PreparePipelineStart(); + private: //------------------------------------------------------------------------ @@ -487,6 +492,10 @@ namespace XrdCl if( !operation ) std::logic_error( "Empty pipeline!" ); Operation *opr = operation.release(); + PipelineHandler *h = opr->handler.get(); + if( h ) + h->PreparePipelineStart(); + opr->Run( timeout, std::move( prms ), std::move( final ) ); } diff --git a/src/XrdCl/XrdClParallelOperation.hh b/src/XrdCl/XrdClParallelOperation.hh index 6736fefbaf0..dd38c56d46b 100644 --- a/src/XrdCl/XrdClParallelOperation.hh +++ b/src/XrdCl/XrdClParallelOperation.hh @@ -313,7 +313,7 @@ namespace XrdCl // although we might have the minimum to succeed we wait for the rest if( status.IsOK() ) return ( pending == 0 ); size_t nb = failed_cnt.fetch_add( 1, std::memory_order_relaxed ); - if( nb + 1 == failed_threshold ) res = status; // we dropped below the threshold + if( nb == failed_threshold ) res = status; // we dropped below the threshold // if we still have to wait for pending operations return false, // otherwise all is done, return true return ( pending == 0 ); diff --git a/src/XrdCl/XrdClPlugInManager.cc b/src/XrdCl/XrdClPlugInManager.cc index 5d8cc00f9d6..3fdaf149630 100644 --- a/src/XrdCl/XrdClPlugInManager.cc +++ b/src/XrdCl/XrdClPlugInManager.cc @@ -459,7 +459,9 @@ namespace XrdCl } std::sort( normalizedURLs.begin(), normalizedURLs.end() ); - std::unique( normalizedURLs.begin(), normalizedURLs.end() ); + + auto last = std::unique( normalizedURLs.begin(), normalizedURLs.end() ); + normalizedURLs.erase( last, normalizedURLs.end() ); if( normalizedURLs.empty() ) return false; diff --git a/src/XrdCl/XrdClPostMaster.cc b/src/XrdCl/XrdClPostMaster.cc index 7b860c3c18c..f75dc911e2d 100644 --- a/src/XrdCl/XrdClPostMaster.cc +++ b/src/XrdCl/XrdClPostMaster.cc @@ -187,15 +187,15 @@ namespace XrdCl //---------------------------------------------------------------------------- bool PostMaster::Stop() { - if( !pImpl->pInitialized ) + if( !pImpl->pInitialized || !pImpl->pRunning ) return true; if( !pImpl->pJobManager->Stop() ) return false; - if( !pImpl->pTaskManager->Stop() ) - return false; if( !pImpl->pPoller->Stop() ) return false; + if( !pImpl->pTaskManager->Stop() ) + return false; pImpl->pRunning = false; return true; } @@ -245,11 +245,15 @@ namespace XrdCl AnyObject &result ) { XrdSysRWLockHelper scopedLock( pImpl->pDisconnectLock ); - PostMasterImpl::ChannelMap::iterator it = - pImpl->pChannelMap.find( url.GetChannelId() ); - if( it == pImpl->pChannelMap.end() ) - return Status( stError, errInvalidOp ); - Channel *channel = it->second; + Channel *channel = 0; + { + XrdSysMutexHelper scopedLock2( pImpl->pChannelMapMutex ); + PostMasterImpl::ChannelMap::iterator it = + pImpl->pChannelMap.find( url.GetChannelId() ); + if( it == pImpl->pChannelMap.end() ) + return Status( stError, errInvalidOp ); + channel = it->second; + } if( !channel ) return Status( stError, errNotSupported ); diff --git a/src/XrdCl/XrdClSIDManager.cc b/src/XrdCl/XrdClSIDManager.cc index 2a40c935067..8718d0eabac 100644 --- a/src/XrdCl/XrdClSIDManager.cc +++ b/src/XrdCl/XrdClSIDManager.cc @@ -18,6 +18,8 @@ #include "XrdCl/XrdClSIDManager.hh" +#include + namespace XrdCl { //---------------------------------------------------------------------------- @@ -48,6 +50,7 @@ namespace XrdCl } memcpy( sid, &allocSID, 2 ); + pAllocTime[allocSID] = time(0); return Status(); } @@ -60,6 +63,7 @@ namespace XrdCl uint16_t relSID = 0; memcpy( &relSID, sid, 2 ); pFreeSIDs.push_back( relSID ); + pAllocTime.erase( relSID ); } //---------------------------------------------------------------------------- @@ -71,6 +75,20 @@ namespace XrdCl uint16_t tiSID = 0; memcpy( &tiSID, sid, 2 ); pTimeOutSIDs.insert( tiSID ); + pAllocTime.erase( tiSID ); + } + + //---------------------------------------------------------------------------- + // Check if any SID was allocated at or before a given time + //---------------------------------------------------------------------------- + bool SIDManager::IsAnySIDOldAs( const time_t tlim ) const + { + XrdSysMutexHelper scopedLock( pMutex ); + return std::any_of( pAllocTime.begin(), pAllocTime.end(), + [tlim](const auto& p) + { + return p.second <= tlim; + } ); } //---------------------------------------------------------------------------- diff --git a/src/XrdCl/XrdClSIDManager.hh b/src/XrdCl/XrdClSIDManager.hh index 0fcdb11f5ce..0eccf4b7098 100644 --- a/src/XrdCl/XrdClSIDManager.hh +++ b/src/XrdCl/XrdClSIDManager.hh @@ -83,6 +83,11 @@ namespace XrdCl //------------------------------------------------------------------------ void TimeOutSID( uint8_t sid[2] ); + //---------------------------------------------------------------------------- + //! Check if any SID was allocated at or before a given time + //---------------------------------------------------------------------------- + bool IsAnySIDOldAs( const time_t tlim ) const; + //------------------------------------------------------------------------ //! Check if a SID is timed out //------------------------------------------------------------------------ @@ -113,6 +118,7 @@ namespace XrdCl uint16_t GetNumberOfAllocatedSIDs() const; private: + std::unordered_map pAllocTime; std::list pFreeSIDs; std::set pTimeOutSIDs; uint16_t pSIDCeiling; diff --git a/src/XrdCl/XrdClSocket.cc b/src/XrdCl/XrdClSocket.cc index a94eb4b486a..a2cb47c127e 100644 --- a/src/XrdCl/XrdClSocket.cc +++ b/src/XrdCl/XrdClSocket.cc @@ -781,7 +781,8 @@ namespace XrdCl //------------------------------------------------------------------------ XRootDStatus Socket::Cork() { -#if defined(TCP_CORK) // it's not defined on mac, we might want explore the possibility of using TCP_NOPUSH +#if defined(TCP_CORK) && !defined(__GNU__) + // it's not defined on mac, we might want explore the possibility of using TCP_NOPUSH if( pCorked ) return XRootDStatus(); int state = 1; @@ -798,7 +799,8 @@ namespace XrdCl //------------------------------------------------------------------------ XRootDStatus Socket::Uncork() { -#if defined(TCP_CORK) // it's not defined on mac, we might want explore the possibility of using TCP_NOPUSH +#if defined(TCP_CORK) && !defined(__GNU__) + // it's not defined on mac, we might want explore the possibility of using TCP_NOPUSH if( !pCorked ) return XRootDStatus(); int state = 0; diff --git a/src/XrdCl/XrdClStream.cc b/src/XrdCl/XrdClStream.cc index b3c4466da35..d9e2681fe0c 100644 --- a/src/XrdCl/XrdClStream.cc +++ b/src/XrdCl/XrdClStream.cc @@ -1024,13 +1024,13 @@ namespace XrdCl //---------------------------------------------------------------------------- // Call back when a message has been reconstructed //---------------------------------------------------------------------------- - void Stream::OnReadTimeout( uint16_t substream ) + bool Stream::OnReadTimeout( uint16_t substream ) { //-------------------------------------------------------------------------- // We only take the main stream into account //-------------------------------------------------------------------------- if( substream != 0 ) - return; + return true; //-------------------------------------------------------------------------- // Check if there is no outgoing messages and if the stream TTL is elapesed. @@ -1070,7 +1070,7 @@ namespace XrdCl // object that aggregates this Stream. //---------------------------------------------------------------------- DefaultEnv::GetPostMaster()->ForceDisconnect( *pUrl ); - return; + return false; } } @@ -1083,14 +1083,17 @@ namespace XrdCl { scopedLock.UnLock(); OnError( substream, st ); + return false; } + return true; } //---------------------------------------------------------------------------- // Call back when a message has been reconstru //---------------------------------------------------------------------------- - void Stream::OnWriteTimeout( uint16_t /*substream*/ ) + bool Stream::OnWriteTimeout( uint16_t /*substream*/ ) { + return true; } //---------------------------------------------------------------------------- diff --git a/src/XrdCl/XrdClStream.hh b/src/XrdCl/XrdClStream.hh index 19522d37de9..5ded1166762 100644 --- a/src/XrdCl/XrdClStream.hh +++ b/src/XrdCl/XrdClStream.hh @@ -31,6 +31,7 @@ #include "XrdSys/XrdSysPthread.hh" #include "XrdSys/XrdSysRAtomic.hh" #include "XrdNet/XrdNetAddr.hh" +#include "XrdOuc/XrdOucCompiler.hh" #include #include #include @@ -219,12 +220,12 @@ namespace XrdCl //------------------------------------------------------------------------ //! On read timeout //------------------------------------------------------------------------ - void OnReadTimeout( uint16_t subStream ); + bool OnReadTimeout( uint16_t subStream ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! On write timeout //------------------------------------------------------------------------ - void OnWriteTimeout( uint16_t subStream ); + bool OnWriteTimeout( uint16_t subStream ) XRD_WARN_UNUSED_RESULT; //------------------------------------------------------------------------ //! Register channel event handler diff --git a/src/XrdCl/XrdClThirdPartyCopyJob.cc b/src/XrdCl/XrdClThirdPartyCopyJob.cc index d3fae4ac1e9..f853470080f 100644 --- a/src/XrdCl/XrdClThirdPartyCopyJob.cc +++ b/src/XrdCl/XrdClThirdPartyCopyJob.cc @@ -348,6 +348,9 @@ namespace XrdCl XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); env->GetInt( "SubStreamsPerChannel", nbStrm ); + // account for the control stream + if (nbStrm > 0) --nbStrm; + bool tpcLiteOnly = false; if( !delegate ) diff --git a/src/XrdCl/XrdClTls.cc b/src/XrdCl/XrdClTls.cc index 115ee0a4d7d..a788b7c2229 100644 --- a/src/XrdCl/XrdClTls.cc +++ b/src/XrdCl/XrdClTls.cc @@ -25,10 +25,13 @@ #include "XrdTls/XrdTls.hh" #include "XrdTls/XrdTlsContext.hh" +#include "XrdOuc/XrdOucUtils.hh" #include #include +static std::unique_ptr tlsContext = nullptr; + namespace { //------------------------------------------------------------------------ @@ -85,21 +88,52 @@ namespace return XrdTls::dbgOFF; } }; - - //------------------------------------------------------------------------ - // Helper function for setting the CA directory in TLS context - //------------------------------------------------------------------------ - static const char* GetCaDir() - { - static const char *envval = getenv("X509_CERT_DIR"); - static const std::string cadir = envval ? envval : - "/etc/grid-security/certificates"; - return cadir.c_str(); - } } namespace XrdCl { + bool InitTLS() + { + if (tlsContext) + return true; + + XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); + XrdCl::Log *log = XrdCl::DefaultEnv::GetLog(); + + int notls = false; + env->GetInt("NoTlsOK", notls); + + if (notls) + return false; + + const char *cadir = getenv("X509_CERT_DIR"); + const char *cafile = getenv("X509_CERT_FILE"); + + if (!cadir && !cafile) + cadir = "/etc/grid-security/certificates"; + + const char *msg; + const mode_t camode = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; + + if (cadir && (msg = XrdOucUtils::ValPath(cadir, camode, true))) { + log->Error(XrdCl::TlsMsg, "Failed to initialize TLS context: CA directory %s", msg); + env->PutInt("NoTlsOK", 1); + return false; + } + + std::string emsg = "unknown error"; + tlsContext = std::make_unique(nullptr, nullptr, cadir, cafile, 0ul, &emsg); + + if (!tlsContext || !tlsContext->isOK()) { + tlsContext.reset(nullptr); + log->Error(XrdCl::TlsMsg, "Failed to initialize TLS context: %s", emsg.c_str()); + env->PutInt("NoTlsOK", 1); + return false; + } + + return true; + } + //------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------ @@ -109,20 +143,12 @@ namespace XrdCl // Set the message callback for TLS layer //---------------------------------------------------------------------- SetTlsMsgCB::Once(); - //---------------------------------------------------------------------- - // we only need one instance of TLS - //---------------------------------------------------------------------- - std::string emsg; - static XrdTlsContext tlsContext( 0, 0, GetCaDir(), 0, 0, &emsg ); - //---------------------------------------------------------------------- - // If the context is not valid throw an exception! We throw generic - // exception as this will be translated to TlsError anyway. - //---------------------------------------------------------------------- - if( !tlsContext.isOK() ) throw std::runtime_error( emsg ); + if( !InitTLS() ) + throw std::runtime_error( "Failed to initialize TLS" ); pTls.reset( - new XrdTlsSocket( tlsContext, pSocket->GetFD(), XrdTlsSocket::TLS_RNB_WNB, + new XrdTlsSocket( *tlsContext, pSocket->GetFD(), XrdTlsSocket::TLS_RNB_WNB, XrdTlsSocket::TLS_HS_NOBLK, true ) ); } diff --git a/src/XrdCl/XrdClTls.hh b/src/XrdCl/XrdClTls.hh index 1b8cc4563d9..f74ce3ecacb 100644 --- a/src/XrdCl/XrdClTls.hh +++ b/src/XrdCl/XrdClTls.hh @@ -30,6 +30,9 @@ namespace XrdCl { class Socket; + /** Initialize TLS context, returns false on failure */ + bool InitTLS(); + //---------------------------------------------------------------------------- //! TLS layer for socket connection //---------------------------------------------------------------------------- diff --git a/src/XrdCl/XrdClUtils.cc b/src/XrdCl/XrdClUtils.cc index 38a02f771a1..2a8d2b4943d 100644 --- a/src/XrdCl/XrdClUtils.cc +++ b/src/XrdCl/XrdClUtils.cc @@ -865,4 +865,54 @@ namespace XrdCl if( dst_supported.count( *itr ) ) return *itr; return std::string(); } + + //---------------------------------------------------------------------------- + //! Split chunks in a ChunkList into one or more ChunkLists + //---------------------------------------------------------------------------- + void Utils::SplitChunks( std::vector &listsvec, + const ChunkList &chunks, + const uint32_t maxcs, + const size_t maxc ) + { + listsvec.clear(); + if( !chunks.size() ) return; + + listsvec.emplace_back(); + ChunkList *c = &listsvec.back(); + const size_t cs = chunks.size(); + size_t idx = 0; + size_t nc = 0; + ChunkInfo tmpc; + + c->reserve( cs ); + + while( idx < cs ) + { + if( maxc && nc >= maxc ) + { + listsvec.emplace_back(); + c = &listsvec.back(); + c->reserve( cs - idx ); + nc = 0; + } + + if( tmpc.length == 0 ) + tmpc = chunks[idx]; + + if( maxcs && tmpc.length > maxcs ) + { + c->emplace_back( tmpc.offset, maxcs, tmpc.buffer ); + tmpc.offset += maxcs; + tmpc.length -= maxcs; + tmpc.buffer = static_cast( tmpc.buffer ) + maxcs; + } + else + { + c->emplace_back( tmpc.offset, tmpc.length, tmpc.buffer ); + tmpc.length = 0; + ++idx; + } + ++nc; + } + } } diff --git a/src/XrdCl/XrdClUtils.hh b/src/XrdCl/XrdClUtils.hh index 6691196f66b..e3c6a502827 100644 --- a/src/XrdCl/XrdClUtils.hh +++ b/src/XrdCl/XrdClUtils.hh @@ -272,6 +272,18 @@ namespace XrdCl if( !st.IsOK() ) return false; return protver >= kXR_PROTPGRWVERSION; } + + //------------------------------------------------------------------------ + //! Split chunks in a ChunkList into one or more ChunkLists + //! @param listsvec : output vector of ChunkLists + //! @param chunks : input ChunkLisits + //! @param maxcs : maximum size of a ChunkInfo in output + //! @param maxc : maximum number of ChunkInfo in each ChunkList + //------------------------------------------------------------------------ + static void SplitChunks( std::vector &listsvec, + const ChunkList &chunks, + const uint32_t maxcs, + const size_t maxc ); }; //---------------------------------------------------------------------------- diff --git a/src/XrdCl/XrdClXRootDMsgHandler.cc b/src/XrdCl/XrdClXRootDMsgHandler.cc index 1bd60a3e336..895784bbfba 100644 --- a/src/XrdCl/XrdClXRootDMsgHandler.cc +++ b/src/XrdCl/XrdClXRootDMsgHandler.cc @@ -904,7 +904,7 @@ namespace XrdCl log->Dump( XRootDMsg, "[%s] Message %s has been successfully sent.", pUrl.GetHostId().c_str(), message->GetDescription().c_str() ); - log->Debug( ExDbgMsg, "[%s] Moving MsgHandler: 0x%x (message: %s ) from out-queu to in-queue.", + log->Debug( ExDbgMsg, "[%s] Moving MsgHandler: 0x%x (message: %s ) from out-queue to in-queue.", pUrl.GetHostId().c_str(), this, pRequest->GetDescription().c_str() ); @@ -1202,7 +1202,9 @@ namespace XrdCl if( pStatus.code == errErrorResponse ) { st->errNo = rsp->body.error.errnum; - std::string errmsg( rsp->body.error.errmsg, rsp->hdr.dlen-4 ); + // omit the last character as the string returned from the server + // (acording to protocol specs) should be null-terminated + std::string errmsg( rsp->body.error.errmsg, rsp->hdr.dlen-5 ); if( st->errNo == kXR_noReplicas && !pLastError.IsOK() ) errmsg += " Last seen error: " + pLastError.ToString(); st->SetErrorMessage( errmsg ); @@ -2443,10 +2445,10 @@ namespace XrdCl { if( sizeof( T ) > buflen ) return Status( stError, errDataError ); - result = *reinterpret_cast( buffer ); + memcpy(&result, buffer, sizeof(T)); buffer += sizeof( T ); - buflen -= sizeof( T ); + buflen -= sizeof( T ); return Status(); } diff --git a/src/XrdCl/XrdClXRootDTransport.cc b/src/XrdCl/XrdClXRootDTransport.cc index f869327aa33..0be81f902ba 100644 --- a/src/XrdCl/XrdClXRootDTransport.cc +++ b/src/XrdCl/XrdClXRootDTransport.cc @@ -795,20 +795,22 @@ namespace XrdCl XrdSysMutexHelper scopedLock( info->mutex ); - uint16_t allocatedSIDs = info->sidManager->GetNumberOfAllocatedSIDs(); + const time_t now = time(0); + const bool anySID = + info->sidManager->IsAnySIDOldAs( now - streamTimeout ); log->Dump( XRootDTransportMsg, "[%s] Stream inactive since %d seconds, " - "stream timeout: %d, allocated SIDs: %d, wait barrier: %s", + "stream timeout: %d, any SID: %d, wait barrier: %s", info->streamName.c_str(), inactiveTime, streamTimeout, - allocatedSIDs, Utils::TimeToString(info->waitBarrier).c_str() ); + anySID, Utils::TimeToString(info->waitBarrier).c_str() ); if( inactiveTime < streamTimeout ) return Status(); - if( time(0) < info->waitBarrier ) + if( now < info->waitBarrier ) return Status(); - if( !allocatedSIDs ) + if( !anySID ) return Status(); return Status( stError, errSocketTimeout ); @@ -1758,6 +1760,13 @@ namespace XrdCl XRootDChannelInfo *info = 0; channelData.Get( info ); + XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); + int notlsok = DefaultNoTlsOK; + env->GetInt( "NoTlsOK", notlsok ); + + if( notlsok ) + return info->encrypted; + // Did the server instructed us to switch to TLS right away? if( info->serverFlags & kXR_gotoTLS ) { @@ -1894,21 +1903,26 @@ namespace XrdCl request->requestid = htons(kXR_protocol); request->clientpv = htonl(kXR_PROTOCOLVERSION); request->flags = ClientProtocolRequest::kXR_secreqs | - ClientProtocolRequest::kXR_bifreqs | - ClientProtocolRequest::kXR_ableTLS; + ClientProtocolRequest::kXR_bifreqs; - bool nodata = false; - if( expect & ClientProtocolRequest::kXR_ExpBind ) - { - XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); - int value = DefaultTlsNoData; - env->GetInt( "TlsNoData", value ); - nodata = bool( value ); - } + int notlsok = DefaultNoTlsOK; + int tlsnodata = DefaultTlsNoData; - if( info->encrypted && !nodata ) + XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv(); + + env->GetInt( "NoTlsOK", notlsok ); + + if (expect & ClientProtocolRequest::kXR_ExpBind) + env->GetInt( "TlsNoData", tlsnodata ); + + if (info->encrypted || InitTLS()) + request->flags |= ClientProtocolRequest::kXR_ableTLS; + + if (info->encrypted && !(notlsok || tlsnodata)) request->flags |= ClientProtocolRequest::kXR_wantTLS; + request->expect = expect; + //-------------------------------------------------------------------------- // If we are in the curse of establishing a connection in the context of // TPC update the expect! (this will be never followed be a bind) @@ -2621,9 +2635,7 @@ namespace XrdCl // credentials //-------------------------------------------------------------------------- XrdNetAddr &srvAddrInfo = *const_cast(hsData->serverAddr); - if( info->encrypted || ( info->serverFlags & kXR_gotoTLS ) || - ( info->serverFlags & kXR_tlsLogin ) ) - srvAddrInfo.SetTLS( true ); + srvAddrInfo.SetTLS( info->encrypted ); while(1) { //------------------------------------------------------------------------ diff --git a/src/XrdCl/XrdClZipArchive.cc b/src/XrdCl/XrdClZipArchive.cc index cba1e464d2f..fb303d23668 100644 --- a/src/XrdCl/XrdClZipArchive.cc +++ b/src/XrdCl/XrdClZipArchive.cc @@ -28,6 +28,7 @@ #include "XrdCl/XrdClLog.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClConstants.hh" +#include "XrdCl/XrdClUtils.hh" #include "XrdZip/XrdZipZIP64EOCDL.hh" #include @@ -499,7 +500,7 @@ namespace XrdCl { // the file does not exist in the archive so it only makes sense // if our user is opening for append - if( flags | OpenFlags::New ) + if( flags & OpenFlags::New ) { openfn = fn; lfh.reset( new LFH( fn, crc32, size, time( 0 ) ) ); @@ -626,10 +627,18 @@ namespace XrdCl } auto wrtbuff = std::make_shared( GetCD() ); - chunks.emplace_back( cdoff, wrtbuff->size(), wrtbuff->data() ); + Pipeline p = XrdCl::Write( archive, cdoff, + wrtbuff->size(), + wrtbuff->data() ); wrtbufs.emplace_back( std::move( wrtbuff ) ); - Pipeline p = XrdCl::VectorWrite( archive, chunks ); + std::vector listsvec; + XrdCl::Utils::SplitChunks( listsvec, chunks, 262144, 1024 ); + + for(auto itr = listsvec.rbegin(); itr != listsvec.rend(); ++itr) + { + p = XrdCl::VectorWrite( archive, *itr ) | p; + } if( ckpinit ) p |= XrdCl::Checkpoint( archive, ChkPtCode::COMMIT ); p |= Close( archive ) >> diff --git a/src/XrdCl/XrdClZipArchive.hh b/src/XrdCl/XrdClZipArchive.hh index d0ed61a2707..5c1acdb7625 100644 --- a/src/XrdCl/XrdClZipArchive.hh +++ b/src/XrdCl/XrdClZipArchive.hh @@ -44,6 +44,7 @@ //----------------------------------------------------------------------------- namespace XrdEc{ class StrmWriter; class Reader; template class OpenOnlyImpl; }; class MicroTest; +class XrdEcTests; namespace XrdCl { @@ -63,6 +64,7 @@ namespace XrdCl template friend class XrdEc::OpenOnlyImpl; friend class ::MicroTest; + friend class ::XrdEcTests; template friend XRootDStatus ReadFromImpl( ZipArchive&, const std::string&, uint64_t, uint32_t, void*, ResponseHandler*, uint16_t ); diff --git a/src/XrdClHttp/CMakeLists.txt b/src/XrdClHttp/CMakeLists.txt index 4af924bfd0c..5e4ceb0ad2e 100644 --- a/src/XrdClHttp/CMakeLists.txt +++ b/src/XrdClHttp/CMakeLists.txt @@ -1,5 +1,3 @@ -include_directories(${Davix_INCLUDE_DIRS}/davix) - set(libXrdClHttp_sources XrdClHttpPlugInFactory.cc XrdClHttpPlugInUtil.cc @@ -11,6 +9,6 @@ set(PLUGIN_NAME "XrdClHttp-${PLUGIN_VERSION}") add_library(${PLUGIN_NAME} MODULE ${libXrdClHttp_sources}) -target_link_libraries(${PLUGIN_NAME} ${Davix_LIBRARIES} XrdCl XrdUtils) +target_link_libraries(${PLUGIN_NAME} PRIVATE Davix::Davix XrdCl XrdUtils) install(TARGETS ${PLUGIN_NAME} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/src/XrdClHttp/XrdClHttpFilePlugIn.cc b/src/XrdClHttp/XrdClHttpFilePlugIn.cc index c58e7f600b9..092d42f1eb3 100644 --- a/src/XrdClHttp/XrdClHttpFilePlugIn.cc +++ b/src/XrdClHttp/XrdClHttpFilePlugIn.cc @@ -375,6 +375,19 @@ XRootDStatus HttpFilePlugIn::Write(uint64_t offset, uint32_t size, return XRootDStatus(); } +//------------------------------------------------------------------------ +//! @see XrdCl::File::PgWrite +//------------------------------------------------------------------------ +XRootDStatus HttpFilePlugIn::PgWrite( uint64_t offset, + uint32_t size, + const void *buffer, + std::vector &cksums, + ResponseHandler *handler, + uint16_t timeout ) +{ (void)cksums; + return Write(offset, size, buffer, handler, timeout); +} + XRootDStatus HttpFilePlugIn::Sync(ResponseHandler *handler, uint16_t timeout) { (void)handler; (void)timeout; @@ -384,6 +397,7 @@ XRootDStatus HttpFilePlugIn::Sync(ResponseHandler *handler, uint16_t timeout) { return XRootDStatus(); } + XRootDStatus HttpFilePlugIn::VectorRead(const ChunkList &chunks, void *buffer, ResponseHandler *handler, uint16_t /*timeout*/) { diff --git a/src/XrdClHttp/XrdClHttpFilePlugIn.hh b/src/XrdClHttp/XrdClHttpFilePlugIn.hh index bde85da5da5..dbab10a603f 100644 --- a/src/XrdClHttp/XrdClHttpFilePlugIn.hh +++ b/src/XrdClHttp/XrdClHttpFilePlugIn.hh @@ -84,6 +84,16 @@ class HttpFilePlugIn : public FilePlugIn { ResponseHandler *handler, uint16_t timeout ) override; + //------------------------------------------------------------------------ + //! @see XrdCl::File::PgWrite - async + //------------------------------------------------------------------------ + virtual XRootDStatus PgWrite( uint64_t offset, + uint32_t size, + const void *buffer, + std::vector &cksums, + ResponseHandler *handler, + uint16_t timeout ) override; + //------------------------------------------------------------------------ //! @see XrdCl::File::Sync //------------------------------------------------------------------------ diff --git a/src/XrdCms/XrdCmsBaseFS.cc b/src/XrdCms/XrdCmsBaseFS.cc index bf0232667d3..40e02df5a19 100644 --- a/src/XrdCms/XrdCmsBaseFS.cc +++ b/src/XrdCms/XrdCmsBaseFS.cc @@ -91,7 +91,7 @@ int XrdCmsBaseFS::Bypass() if (Interval >= 450) {theQ.rLeft = theQ.rAgain; Window.Reset(); - cerr <<"BYPASS " <NodeMask & mask)) {if (!(selR.needNet & np->hasNet)) {selR.xNoNet= true; continue;} selR.nPick++; @@ -1800,7 +1807,17 @@ XrdCmsNode *XrdCmsCluster::SelbyLoad(SMask_t mask, XrdCmsSelector &selR) || (reqSS && np->isNoStage))) {selR.xFull = true; continue;} if (!sp) sp = np; - else{if (selR.needSpace) + else{ + if (Config.P_randlb==1){ + //add 1 to the inverse load, this is to allow some selection in case reported loads hit 100 + weighed[i] = selCap + static_cast(101 - np->myLoad + + std::pow(101 - np->myLoad, + std::log(100)-std::log(std::min(std::max(Config.P_fuzz,1),100)))/2); + selCap += static_cast(101 - np->myLoad + + std::pow(101 - np->myLoad, + std::log(100)-std::log(std::min(std::max(Config.P_fuzz,1),100)))/2); + } + else{if (selR.needSpace) {if (abs(sp->myMass - np->myMass) <= Config.P_fuzz) {if (sp->RefW > (np->RefW+Config.DiskLinger)) sp=np;} else if (sp->myMass > np->myMass) sp=np; @@ -1816,12 +1833,29 @@ XrdCmsNode *XrdCmsCluster::SelbyLoad(SMask_t mask, XrdCmsSelector &selR) } Multi = true; } + } } - + if (Config.P_randlb==1){ + // pick a random weighed node + // + std::random_device rand_dev; + std::mt19937 generator(rand_dev()); + std::uniform_int_distribution distr(randomSel,selCap); + randomSel = distr(generator); + for(int i=0;i<=STHi;i++){ + if(randomSel<=weighed[i]){ + sp=NodeTab[i]; + break; + } + } + } + delete [] weighed; // Check for overloaded node and return result // if (!sp) return calcDelay(selR); + if (Config.P_randlb!=1){ RefCount(sp, Multi, selR.needSpace); + } return sp; } diff --git a/src/XrdCms/XrdCmsConfig.cc b/src/XrdCms/XrdCmsConfig.cc index 9512df778d6..ca47b3ab6a5 100644 --- a/src/XrdCms/XrdCmsConfig.cc +++ b/src/XrdCms/XrdCmsConfig.cc @@ -735,6 +735,7 @@ void XrdCmsConfig::ConfigDefaults(void) P_load = 0; P_mem = 0; P_pag = 0; + P_randlb = 0; // SelbyLoad algoruthm choice AskPerf = 10; // Every 10 pings AskPing = 60; // Every 1 minute PingTick = 0; @@ -1273,7 +1274,7 @@ char *XrdCmsConfig::setupSid() void XrdCmsConfig::Usage(int rc) { -cerr <<"\nUsage: cmsd [xrdopts] [-i] [-m] [-s] -c " <" <Locate(Resp, newPath.c_str(), flags, EnvInfo); // set new error message to full url:port//newPath - const std::string errText { std::string(Resp.getErrText()) + ':' + to_string(Resp.getErrInfo()) + newPath}; + const std::string errText { std::string(Resp.getErrText()) + ':' + std::to_string(Resp.getErrInfo()) + newPath}; Resp.setErrInfo(0, errText.c_str()); // now have normal redirection to dataserver at url:port return rcode; diff --git a/src/XrdCrypto.cmake b/src/XrdCrypto.cmake index e820e433e4f..fecb569c1cd 100644 --- a/src/XrdCrypto.cmake +++ b/src/XrdCrypto.cmake @@ -1,4 +1,3 @@ -include( XRootDCommon ) #------------------------------------------------------------------------------- # Modules @@ -39,6 +38,7 @@ add_library( target_link_libraries( XrdCrypto + PRIVATE XrdUtils ${CMAKE_DL_LIBS} ) @@ -46,9 +46,7 @@ set_target_properties( XrdCrypto PROPERTIES VERSION ${XRD_CRYPTO_VERSION} - SOVERSION ${XRD_CRYPTO_SOVERSION} - INTERFACE_LINK_LIBRARIES "" - LINK_INTERFACE_LIBRARIES "" ) + SOVERSION ${XRD_CRYPTO_SOVERSION} ) #------------------------------------------------------------------------------- # The XrdCryptoLite library @@ -65,6 +63,7 @@ add_library( target_link_libraries( XrdCryptoLite + PRIVATE XrdUtils OpenSSL::Crypto ) @@ -72,9 +71,7 @@ set_target_properties( XrdCryptoLite PROPERTIES VERSION ${XRD_CRYPTO_LITE_VERSION} - SOVERSION ${XRD_CRYPTO_LITE_SOVERSION} - INTERFACE_LINK_LIBRARIES "" - LINK_INTERFACE_LIBRARIES "" ) + SOVERSION ${XRD_CRYPTO_LITE_SOVERSION} ) #------------------------------------------------------------------------------- # The XrdCryptossl module @@ -99,17 +96,12 @@ add_library( target_link_libraries( ${LIB_XRD_CRYPTOSSL} + PRIVATE XrdCrypto XrdUtils ${CMAKE_THREAD_LIBS_INIT} OpenSSL::SSL ) -set_target_properties( - ${LIB_XRD_CRYPTOSSL} - PROPERTIES - INTERFACE_LINK_LIBRARIES "" - LINK_INTERFACE_LIBRARIES "" ) - #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/src/XrdCrypto/XrdCryptoAux.hh b/src/XrdCrypto/XrdCryptoAux.hh index b9555ad5749..cbc810ff7ff 100644 --- a/src/XrdCrypto/XrdCryptoAux.hh +++ b/src/XrdCrypto/XrdCryptoAux.hh @@ -38,7 +38,7 @@ /******************************************************************************/ /* M i s c e l l a n e o u s D e f i n e s */ /******************************************************************************/ -#define ABSTRACTMETHOD(x) {cerr <<"Method "< plugins; @@ -426,6 +428,10 @@ XrdCryptoFactory *XrdCryptoFactory::GetCryptoFactory(const char *factoryid) char factobjname[80], libfn[80]; EPNAME("Factory::GetCryptoFactory"); + // Factory entries are tracked in a static list. + // Make sure only one thread may be using or modifying the list at a time. + XrdSysMutexHelper mHelp(fMutex); + // // The id must be defined if (!factoryid) { diff --git a/src/XrdCrypto/XrdCryptoLite_bf32.cc b/src/XrdCrypto/XrdCryptoLite_bf32.cc index 9a738234462..72f05a99c8f 100644 --- a/src/XrdCrypto/XrdCryptoLite_bf32.cc +++ b/src/XrdCrypto/XrdCryptoLite_bf32.cc @@ -42,6 +42,9 @@ #include #include +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#include "XrdTls/XrdTlsContext.hh" +#endif #if OPENSSL_VERSION_NUMBER >= 0x30000000L #include #endif @@ -178,6 +181,19 @@ int XrdCryptoLite_bf32::Encrypt(const char *key, XrdCryptoLite *XrdCryptoLite_New_bf32(const char Type) { #ifdef HAVE_SSL +#if OPENSSL_VERSION_NUMBER < 0x10100000L + // In case nothing has yet configured a libcrypto thread-id callback + // function we provide one via the XrdTlsContext Init method. Compared + // to the default the aim is to provide better properies when libcrypto + // uses the thread-id as hash-table keys for the per-thread error state. + static struct configThreadid { + configThreadid() {eText = XrdTlsContext::Init();} + const char *eText; + } ctid; + // Make sure all went well + // + if (ctid.eText) return (XrdCryptoLite *)0; +#endif #if OPENSSL_VERSION_NUMBER >= 0x30000000L // With openssl v3 the blowfish cipher is only available via the "legacy" // provider. Legacy is typically not enabled by default (but can be via diff --git a/src/XrdCrypto/XrdCryptoTrace.hh b/src/XrdCrypto/XrdCryptoTrace.hh index b1f4355a84d..3e6ada075b1 100644 --- a/src/XrdCrypto/XrdCryptoTrace.hh +++ b/src/XrdCrypto/XrdCryptoTrace.hh @@ -38,7 +38,7 @@ #define QTRACE(act) (cryptoTrace && (cryptoTrace->What & cryptoTRACE_ ## act)) #define PRINT(y) {if (cryptoTrace) {cryptoTrace->Beg(epname); \ - cerr <End();}} + std::cerr <End();}} #define TRACE(act,x) if (QTRACE(act)) PRINT(x) #define DEBUG(y) TRACE(Debug,y) #define EPNAME(x) static const char *epname = x; diff --git a/src/XrdCrypto/XrdCryptoX509Chain.cc b/src/XrdCrypto/XrdCryptoX509Chain.cc index ad618fb7527..d1adc0bc39f 100644 --- a/src/XrdCrypto/XrdCryptoX509Chain.cc +++ b/src/XrdCrypto/XrdCryptoX509Chain.cc @@ -40,7 +40,7 @@ // ---------------------------------------------------------------------------// // For test dumps, to avoid interfering with the trace mutex -#define LOCDUMP(y) { cerr << epname << ":" << y << endl; } +#define LOCDUMP(y) { std::cerr << epname << ":" << y << std::endl; } // Description of errors static const char *X509ChainErrStr[] = { diff --git a/src/XrdCrypto/XrdCryptosslCipher.cc b/src/XrdCrypto/XrdCryptosslCipher.cc index 2e0e67e7443..ea9c3b78ad3 100644 --- a/src/XrdCrypto/XrdCryptosslCipher.cc +++ b/src/XrdCrypto/XrdCryptosslCipher.cc @@ -47,6 +47,26 @@ #include #endif +// Hardcoded DH parameters that are acceptable to both OpenSSL 3.0 (RHEL9) +// and 1.0.2 (RHEL7). OpenSSL 3.0 reworked the DH parameter generation algorithm +// and now produces DH params that don't pass OpenSSL 1.0.2's parameter verification +// function (`DH_check`). Accordingly, since these are safe to reuse, we generated +// a single set of parameters for the server to always utilize. +static const char dh_param_enc[] = +R"( +-----BEGIN DH PARAMETERS----- +MIIBiAKCAYEAzcEAf3ZCkm0FxJLgKd1YoT16Hietl7QV8VgJNc5CYKmRu/gKylxT +MVZJqtUmoh2IvFHCfbTGEmZM5LdVaZfMLQf7yXjecg0nSGklYZeQQ3P0qshFLbI9 +u3z1XhEeCbEZPq84WWwXacSAAxwwRRrN5nshgAavqvyDiGNi+GqYpqGPb9JE38R3 +GJ51FTPutZlvQvEycjCbjyajhpItBB+XvIjWj2GQyvi+cqB0WrPQAsxCOPrBTCZL +OjM0NfJ7PQfllw3RDQev2u1Q+Rt8QyScJQCFUj/SWoxpw2ydpWdgAkrqTmdVYrev +x5AoXE52cVIC8wfOxaaJ4cBpnJui3Y0jZcOQj0FtC0wf4WcBpHnLLBzKSOQwbxts +WE8LkskPnwwrup/HqWimFFg40bC9F5Lm3CTDCb45mtlBxi3DydIbRLFhGAjlKzV3 +s9G3opHwwfgXpFf3+zg7NPV3g1//HLgWCvooOvMqaO+X7+lXczJJLMafEaarcAya +Kyo8PGKIAORrAgEF +-----END DH PARAMETERS----- +)"; + // ---------------------------------------------------------------------------// // // Cipher interface @@ -146,7 +166,33 @@ static int DSA_set0_key(DSA *d, BIGNUM *pub_key, BIGNUM *priv_key) } #endif +static EVP_PKEY *getFixedDHParams() { + static EVP_PKEY *dhparms = [] { + EVP_PKEY *dhParam = 0; + + BIO *biop = BIO_new(BIO_s_mem()); + BIO_write(biop, dh_param_enc, strlen(dh_param_enc)); + PEM_read_bio_Parameters(biop, &dhParam); + BIO_free(biop); + return dhParam; + }(); + + assert(dhparms); + return dhparms; +} + static int XrdCheckDH (EVP_PKEY *pkey) { + // If the DH parameters we received are our fixed set we know they + // are acceptable. The parameter check requires computation and more + // with openssl 3 than previously. So skip if DH params are known. + const EVP_PKEY *dhparms = getFixedDHParams(); +#if OPENSSL_VERSION_NUMBER >= 0x30000000L + const bool skipcheck = EVP_PKEY_parameters_eq(pkey, dhparms); +#else + const bool skipcheck = EVP_PKEY_cmp_parameters(pkey, dhparms); +#endif + if (skipcheck) return 1; + int rc; #if OPENSSL_VERSION_NUMBER < 0x10101000L DH *dh = EVP_PKEY_get0_DH(pkey); @@ -504,23 +550,43 @@ XrdCryptosslCipher::XrdCryptosslCipher(bool padded, int bits, char *pub, deflength = 1; if (!pub) { - static EVP_PKEY *dhparms = [] { - DEBUG("generate DH parameters"); - EVP_PKEY *dhParam = 0; + + DEBUG("generate DH parameters"); + EVP_PKEY *dhparms = getFixedDHParams(); +// +// Important historical context: +// - We used to generate DH params on every server startup (commented +// out below). This was prohibitively costly to do on startup for +// DH parameters large enough to be considered secure. +// - OpenSSL 3.0 improved the DH parameter generation to avoid leaking +// the first bit of the session key (see https://github.com/openssl/openssl/issues/9792 +// for more information). However, a side-effect is that the new +// parameters are not recognized as valid in OpenSSL 1.0.2. +// - Since we can't control old client versions and new servers can't +// generate compatible DH parameters, we switch to a fixed, much stronger +// set of DH parameters (3072 bits). +// +// The impact is that we continue leaking the first bit of the session key +// (meaning it's effectively 127 bits not 128 bits -- still plenty secure) +// but upgrade the DH parameters to something more modern (3072; previously, +// it was 512 bits which was not considered secure). The downside +// of fixed DH parameters is that if a nation-state attacked our selected +// parameters (using technology not currently available), we would have +// to upgrade all servers with a new set of DH parameters. +// + +/* EVP_PKEY_CTX *pkctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, 0); EVP_PKEY_paramgen_init(pkctx); EVP_PKEY_CTX_set_dh_paramgen_prime_len(pkctx, kDHMINBITS); EVP_PKEY_CTX_set_dh_paramgen_generator(pkctx, 5); EVP_PKEY_paramgen(pkctx, &dhParam); EVP_PKEY_CTX_free(pkctx); - DEBUG("generate DH parameters done"); - return dhParam; - }(); +*/ DEBUG("configure DH parameters"); // // Set params for DH object - assert(dhparms); EVP_PKEY_CTX *pkctx = EVP_PKEY_CTX_new(dhparms, 0); EVP_PKEY_keygen_init(pkctx); EVP_PKEY_keygen(pkctx, &fDH); @@ -853,6 +919,7 @@ bool XrdCryptosslCipher::Finalize(bool padded, EVP_PKEY_derive_set_peer(pkctx, peer); EVP_PKEY_derive(pkctx, (unsigned char *)ktmp, <mp); EVP_PKEY_CTX_free(pkctx); + EVP_PKEY_free(peer); if (ltmp > 0) { #if OPENSSL_VERSION_NUMBER < 0x10101000L if (padded) { @@ -1036,7 +1103,7 @@ void XrdCryptosslCipher::PrintPublic(BIGNUM *pub) char *bpub = new char[lpub]; if (bpub) { BIO_read(biop,(void *)bpub,lpub); - cerr << bpub << endl; + std::cerr << bpub << std::endl; delete[] bpub; } EVP_PKEY_free(dsa); diff --git a/src/XrdCrypto/XrdCryptosslCipher.hh b/src/XrdCrypto/XrdCryptosslCipher.hh index f24fc01508b..853716a6e99 100644 --- a/src/XrdCrypto/XrdCryptosslCipher.hh +++ b/src/XrdCrypto/XrdCryptosslCipher.hh @@ -39,7 +39,12 @@ #include #include -#define kDHMINBITS 512 +// This is not used as we no longer dynamically generate the DH parameters; +// see the comments in XrdCryptosslCipher.cc for more context. +// Purposely keeping it around to help make the issue visible to future readers +// of the code. +// +// #define kDHMINBITS 512 // ---------------------------------------------------------------------------// // diff --git a/src/XrdCrypto/XrdCryptosslRSA.cc b/src/XrdCrypto/XrdCryptosslRSA.cc index dca76f8acaf..54818139ac3 100644 --- a/src/XrdCrypto/XrdCryptosslRSA.cc +++ b/src/XrdCrypto/XrdCryptosslRSA.cc @@ -322,6 +322,8 @@ int XrdCryptosslRSA::ImportPrivate(const char *pri, int lpri) if (!fEVP) return -1; + + int rc = -1; prilen = -1; // Bio for exporting the pub key @@ -337,9 +339,10 @@ int XrdCryptosslRSA::ImportPrivate(const char *pri, int lpri) if (PEM_read_bio_PrivateKey(bpri, &fEVP, 0, 0)) { // Update status status = kComplete; - return 0; + rc = 0; } - return -1; + BIO_free(bpri); + return rc; } //_____________________________________________________________________________ @@ -354,7 +357,7 @@ void XrdCryptosslRSA::Dump() char *btmp = new char[GetPublen()+1]; if (btmp) { ExportPublic(btmp,GetPublen()+1); - DEBUG("export pub key:"<What & cryptoTRACE_ ## act)) #define PRINT(y) {if (sslTrace) {sslTrace->Beg(epname); \ - cerr <End();}} + std::cerr <End();}} #define TRACE(act,x) if (QTRACE(act)) PRINT(x) #define DEBUG(y) TRACE(Debug,y) #define EPNAME(x) static const char *epname = x; diff --git a/src/XrdCrypto/XrdCryptosslX509Crl.cc b/src/XrdCrypto/XrdCryptosslX509Crl.cc index 653aa38902c..6ece76af59d 100644 --- a/src/XrdCrypto/XrdCryptosslX509Crl.cc +++ b/src/XrdCrypto/XrdCryptosslX509Crl.cc @@ -376,6 +376,12 @@ int XrdCryptosslX509Crl::GetFileType(const char *crlfn) return rc; } +bool XrdCryptosslX509Crl::hasCriticalExtension() { + // If the X509_CRL_get_ext_by_critical() function returns -1, no critical extension + // has been found + return X509_CRL_get_ext_by_critical(crl,1,-1) != -1; +} + //_____________________________________________________________________________ int XrdCryptosslX509Crl::LoadCache() { diff --git a/src/XrdCrypto/XrdCryptosslX509Crl.hh b/src/XrdCrypto/XrdCryptosslX509Crl.hh index 26ca673ec6a..d2dc06b66e5 100644 --- a/src/XrdCrypto/XrdCryptosslX509Crl.hh +++ b/src/XrdCrypto/XrdCryptosslX509Crl.hh @@ -83,6 +83,9 @@ public: // Dump CRL object to a file. bool ToFile(FILE *fh); + //Returns true if the CRL certificate has critical extension, false otherwise + bool hasCriticalExtension(); + private: X509_CRL *crl{nullptr}; // The CRL object time_t lastupdate{-1}; // time of last update diff --git a/src/XrdCrypto/XrdCryptosslgsiAux.cc b/src/XrdCrypto/XrdCryptosslgsiAux.cc index 09edd44229d..a88dbd900b8 100644 --- a/src/XrdCrypto/XrdCryptosslgsiAux.cc +++ b/src/XrdCrypto/XrdCryptosslgsiAux.cc @@ -42,6 +42,7 @@ #include #include #include +#include #include "XrdSut/XrdSutRndm.hh" #include "XrdCrypto/XrdCryptogsiX509Chain.hh" @@ -51,6 +52,26 @@ #include "XrdCrypto/XrdCryptosslX509.hh" #include "XrdCrypto/XrdCryptosslX509Req.hh" +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++// +// // +// type aliases to ease use of smart pointers with common ssl structures // +// // +//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++// +static void stackOfX509ExtensionDelete(STACK_OF(X509_EXTENSION) *ske) { +#if OPENSSL_VERSION_NUMBER >= 0x10000000L + sk_X509_EXTENSION_pop_free(ske, X509_EXTENSION_free); +#else /* OPENSSL */ + sk_pop_free(ske, X509_EXTENSION_free); +#endif /* OPENSSL */ +} +using EVP_PKEY_ptr = std::unique_ptr; +using X509_ptr = std::unique_ptr; +using X509_NAME_ptr = std::unique_ptr; +using X509_REQ_ptr = std::unique_ptr; +using X509_EXTENSION_ptr = std::unique_ptr; +using PROXY_CERT_INFO_EXTENSION_ptr = std::unique_ptr; +using STACK_OF_X509_EXTENSION_ptr = std::unique_ptr; + //+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++// // // // Extensions OID relevant for proxies // @@ -455,7 +476,7 @@ int XrdCryptosslX509CreateProxy(const char *fnc, const char *fnk, } // // Sign the request - if (!(X509_REQ_sign(preq, ekPX, EVP_sha1()))) { + if (!(X509_REQ_sign(preq, ekPX, EVP_sha256()))) { PRINT("problems signing the request"); return -kErrPX_Signing; } @@ -549,7 +570,7 @@ int XrdCryptosslX509CreateProxy(const char *fnc, const char *fnk, // // Sign the certificate - if (!(X509_sign(xPX, ekEEC, EVP_sha1()))) { + if (!(X509_sign(xPX, ekEEC, EVP_sha256()))) { PRINT("problems signing the certificate"); return -kErrPX_Signing; } @@ -656,9 +677,20 @@ int XrdCryptosslX509CreateProxyReq(XrdCryptoX509 *xcpi, PRINT("EEC certificate has expired"); return -kErrPX_ExpiredEEC; } + + // These will be assigned dynamically allocated ssl structures later. + // They use type aliases for unique_ptr, to ease use of a smart pointer. + // + EVP_PKEY_ptr ekro(nullptr, &EVP_PKEY_free); + X509_EXTENSION_ptr ext(nullptr, &X509_EXTENSION_free); + X509_NAME_ptr psubj(nullptr, &X509_NAME_free); + X509_REQ_ptr xro(nullptr, &X509_REQ_free); + PROXY_CERT_INFO_EXTENSION_ptr pci(nullptr, &PROXY_CERT_INFO_EXTENSION_free); + STACK_OF_X509_EXTENSION_ptr esk(nullptr, &stackOfX509ExtensionDelete); + // // Create a new request - X509_REQ *xro = X509_REQ_new(); + xro.reset(X509_REQ_new()); if (!xro) { PRINT("cannot to create cert request"); return -kErrPX_NoResources; @@ -666,7 +698,10 @@ int XrdCryptosslX509CreateProxyReq(XrdCryptoX509 *xcpi, // // Use same num of bits as the signing certificate, but // less than 512 - int bits = EVP_PKEY_bits(X509_get_pubkey(xpi)); + ekro.reset(X509_get_pubkey(xpi)); + int bits = EVP_PKEY_bits(ekro.get()); + ekro = nullptr; + bits = (bits < 512) ? 512 : bits; // // Create the new PKI for the proxy (exponent 65537) @@ -676,7 +711,6 @@ int XrdCryptosslX509CreateProxyReq(XrdCryptoX509 *xcpi, return -kErrPX_GenerateKey; } BN_set_word(e, 0x10001); - EVP_PKEY *ekro = 0; EVP_PKEY_CTX *pkctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, 0); EVP_PKEY_keygen_init(pkctx); EVP_PKEY_CTX_set_rsa_keygen_bits(pkctx, bits); @@ -686,7 +720,11 @@ int XrdCryptosslX509CreateProxyReq(XrdCryptoX509 *xcpi, #else EVP_PKEY_CTX_set_rsa_keygen_pubexp(pkctx, e); #endif - EVP_PKEY_keygen(pkctx, &ekro); + { + EVP_PKEY *tmppk = nullptr; + EVP_PKEY_keygen(pkctx, &tmppk); + ekro.reset(tmppk); + } EVP_PKEY_CTX_free(pkctx); // // Set the key into the request @@ -694,7 +732,7 @@ int XrdCryptosslX509CreateProxyReq(XrdCryptoX509 *xcpi, PRINT("proxy key could not be generated - return"); return -kErrPX_GenerateKey; } - X509_REQ_set_pubkey(xro, ekro); + X509_REQ_set_pubkey(xro.get(), ekro.get()); // // Generate a serial number. Specification says that this *should* // unique, so we just draw an unsigned random integer @@ -704,16 +742,16 @@ int XrdCryptosslX509CreateProxyReq(XrdCryptoX509 *xcpi, // with is a random unsigned int used also as serial // number. // Duplicate user subject name - X509_NAME *psubj = X509_NAME_dup(X509_get_subject_name(xpi)); + psubj.reset(X509_NAME_dup(X509_get_subject_name(xpi))); if (xcro && *xcro && *((int *)(*xcro)) <= 10100) { // Delete existing proxy CN addition; for backward compatibility #if OPENSSL_VERSION_NUMBER >= 0x10000000L - int ne = X509_NAME_entry_count(psubj); + int ne = X509_NAME_entry_count(psubj.get()); #else /* OPENSSL */ int ne = psubj->entries->num; #endif /* OPENSSL */ if (ne >= 0) { - X509_NAME_ENTRY *cne = X509_NAME_delete_entry(psubj, ne-1); + X509_NAME_ENTRY *cne = X509_NAME_delete_entry(psubj.get(), ne-1); if (cne) { X509_NAME_ENTRY_free(cne); } else { @@ -725,21 +763,21 @@ int XrdCryptosslX509CreateProxyReq(XrdCryptoX509 *xcpi, // Create an entry with the common name unsigned char sn[20] = {0}; sprintf((char *)sn, "%d", serial); - if (!X509_NAME_add_entry_by_txt(psubj, (char *)"CN", MBSTRING_ASC, + if (!X509_NAME_add_entry_by_txt(psubj.get(), (char *)"CN", MBSTRING_ASC, sn, -1, -1, 0)) { PRINT("could not add CN - (serial: "<proxyPolicy->policyLanguage = OBJ_txt2obj("1.3.6.1.5.5.7.21.1", 1); // // Create a stack - STACK_OF(X509_EXTENSION) *esk = sk_X509_EXTENSION_new_null(); + esk.reset(sk_X509_EXTENSION_new_null()); if (!esk) { PRINT("could not create stack for extensions"); return -kErrPX_NoResources; @@ -780,11 +818,13 @@ int XrdCryptosslX509CreateProxyReq(XrdCryptoX509 *xcpi, inpci->pcPathLengthConstraint) indepthlen = ASN1_INTEGER_get(inpci->pcPathLengthConstraint); DEBUG("IN depth length: "<length = i2d_PROXY_CERT_INFO_EXTENSION(pci, 0); - if (!(X509_EXTENSION_get_data(ext)->data = (unsigned char *)malloc(X509_EXTENSION_get_data(ext)->length+1))) { + X509_EXTENSION_get_data(ext.get())->length = i2d_PROXY_CERT_INFO_EXTENSION(pci.get(), 0); + if (!(X509_EXTENSION_get_data(ext.get())->data = (unsigned char *)malloc(X509_EXTENSION_get_data(ext.get())->length+1))) { PRINT("could not allocate data field for extension"); return -kErrPX_NoResources; } - unsigned char *pp = X509_EXTENSION_get_data(ext)->data; - if ((i2d_PROXY_CERT_INFO_EXTENSION(pci, &pp)) <= 0) { + unsigned char *pp = X509_EXTENSION_get_data(ext.get())->data; + if ((i2d_PROXY_CERT_INFO_EXTENSION(pci.get(), &pp)) <= 0) { PRINT("problem converting data for extension"); return -kErrPX_Error; } + pci = nullptr; + // Set extension name. ASN1_OBJECT *obj = OBJ_txt2obj(gsiProxyCertInfo_OID, 1); - if (!obj || X509_EXTENSION_set_object(ext, obj) != 1) { + if (!obj || X509_EXTENSION_set_object(ext.get(), obj) != 1) { PRINT("could not set extension name"); + ASN1_OBJECT_free(obj); return -kErrPX_SetAttribute; } + ASN1_OBJECT_free(obj); + obj = 0; + // flag as critical - if (X509_EXTENSION_set_critical(ext, 1) != 1) { + if (X509_EXTENSION_set_critical(ext.get(), 1) != 1) { PRINT("could not set extension critical flag"); return -kErrPX_SetAttribute; } - if (sk_X509_EXTENSION_push(esk, ext) == 0) { + if (sk_X509_EXTENSION_push(esk.get(), ext.get()) == 0) { PRINT("could not push the extension in the stack"); return -kErrPX_Error; } + // ext resource now owned by esk + ext.release(); + // Add extensions - if (!(X509_REQ_add_extensions(xro, esk))) { + if (!(X509_REQ_add_extensions(xro.get(), esk.get()))) { PRINT("problem adding extension"); return -kErrPX_SetAttribute; } // // Sign the request - if (!(X509_REQ_sign(xro, ekro, EVP_sha1()))) { + if (!(X509_REQ_sign(xro.get(), ekro.get(), EVP_sha256()))) { PRINT("problems signing the request"); return -kErrPX_Signing; } // Prepare output - *xcro = new XrdCryptosslX509Req(xro); - *kcro = new XrdCryptosslRSA(ekro); + *xcro = new XrdCryptosslX509Req(xro.get()); + *kcro = new XrdCryptosslRSA(ekro.get()); - // Cleanup -#if OPENSSL_VERSION_NUMBER >= 0x10000000L - sk_X509_EXTENSION_pop_free(esk, X509_EXTENSION_free); -#else /* OPENSSL */ - sk_free(esk); -#endif /* OPENSSL */ + // xro, ekro resoruce now owned by *xcro and *kcro + xro.release(); + ekro.release(); // We are done return 0; @@ -900,9 +946,19 @@ int XrdCryptosslX509SignProxyReq(XrdCryptoX509 *xcpi, XrdCryptoRSA *kcpi, PRINT("inconsistent key loaded"); return -kErrPX_BadEECkey; } + + // These will be assigned dynamically allocated ssl structures later. + // They use type aliases for unique_ptr, to ease use of a smart pointer. + // + EVP_PKEY_ptr ekpi(nullptr, &EVP_PKEY_free); + X509_ptr xpo(nullptr, &X509_free); + X509_EXTENSION_ptr ext(nullptr, &X509_EXTENSION_free); + PROXY_CERT_INFO_EXTENSION_ptr pci(nullptr, &PROXY_CERT_INFO_EXTENSION_free); + STACK_OF_X509_EXTENSION_ptr xrisk(nullptr, &stackOfX509ExtensionDelete); + // Point to the cerificate #if OPENSSL_VERSION_NUMBER >= 0x30000000L - EVP_PKEY *ekpi = EVP_PKEY_dup((EVP_PKEY *)(kcpi->Opaque())); + ekpi.reset(EVP_PKEY_dup((EVP_PKEY *)(kcpi->Opaque()))); if (!ekpi) { PRINT("could not create a EVP_PKEY * instance - return"); return -kErrPX_NoResources; @@ -911,12 +967,12 @@ int XrdCryptosslX509SignProxyReq(XrdCryptoX509 *xcpi, XrdCryptoRSA *kcpi, RSA *kpi = EVP_PKEY_get0_RSA((EVP_PKEY *)(kcpi->Opaque())); // // Set the key into the request - EVP_PKEY *ekpi = EVP_PKEY_new(); + ekpi.reset(EVP_PKEY_new()); if (!ekpi) { PRINT("could not create a EVP_PKEY * instance - return"); return -kErrPX_NoResources; } - EVP_PKEY_set1_RSA(ekpi, kpi); + EVP_PKEY_set1_RSA(ekpi.get(), kpi); #endif // Get request in raw form @@ -960,50 +1016,50 @@ int XrdCryptosslX509SignProxyReq(XrdCryptoX509 *xcpi, XrdCryptoRSA *kcpi, unsigned int serial = (unsigned int)(strtol(sserial.c_str(), 0, 10)); // // Create new proxy cert - X509 *xpo = X509_new(); + xpo.reset(X509_new()); if (!xpo) { PRINT("could not create certificate object for proxies"); return -kErrPX_NoResources; } // Set version number - if (X509_set_version(xpo, 2L) != 1) { + if (X509_set_version(xpo.get(), 2L) != 1) { PRINT("could not set version"); return -kErrPX_SetAttribute; } // Set serial number - if (ASN1_INTEGER_set(X509_get_serialNumber(xpo), serial) != 1) { + if (ASN1_INTEGER_set(X509_get_serialNumber(xpo.get()), serial) != 1) { PRINT("could not set serial number"); return -kErrPX_SetAttribute; } // Set subject name - if (X509_set_subject_name(xpo, X509_REQ_get_subject_name(xri)) != 1) { + if (X509_set_subject_name(xpo.get(), X509_REQ_get_subject_name(xri)) != 1) { PRINT("could not set subject name"); return -kErrPX_SetAttribute; } // Set issuer name - if (X509_set_issuer_name(xpo, X509_get_subject_name(xpi)) != 1) { + if (X509_set_issuer_name(xpo.get(), X509_get_subject_name(xpi)) != 1) { PRINT("could not set issuer name"); return -kErrPX_SetAttribute; } // Set public key - if (X509_set_pubkey(xpo, X509_REQ_get_pubkey(xri)) != 1) { + if (X509_set_pubkey(xpo.get(), X509_REQ_get_pubkey(xri)) != 1) { PRINT("could not set public key"); return -kErrPX_SetAttribute; } // Set proxy validity: notBefore now - if (!X509_gmtime_adj(X509_get_notBefore(xpo), 0)) { + if (!X509_gmtime_adj(X509_get_notBefore(xpo.get()), 0)) { PRINT("could not set notBefore"); return -kErrPX_SetAttribute; } // Set proxy validity: notAfter timeleft from now - if (!X509_gmtime_adj(X509_get_notAfter(xpo), timeleft)) { + if (!X509_gmtime_adj(X509_get_notAfter(xpo.get()), timeleft)) { PRINT("could not set notAfter"); return -kErrPX_SetAttribute; } @@ -1033,6 +1089,7 @@ int XrdCryptosslX509SignProxyReq(XrdCryptoX509 *xcpi, XrdCryptoRSA *kcpi, inpci->pcPathLengthConstraint) indepthlen = ASN1_INTEGER_get(inpci->pcPathLengthConstraint); DEBUG("IN depth length: "<= 0x10000000L - int nriext = sk_X509_EXTENSION_num(xrisk); + int nriext = sk_X509_EXTENSION_num(xrisk.get()); #else /* OPENSSL */ - int nriext = sk_num(xrisk); + int nriext = sk_num(xrisk.get()); #endif /* OPENSSL */ if (nriext == 0 || !haskeyusage) { PRINT("wrong extensions in request: "<< nriext<<", "<pcPathLengthConstraint) reqdepthlen = ASN1_INTEGER_get(reqpci->pcPathLengthConstraint); + PROXY_CERT_INFO_EXTENSION_free(reqpci); } DEBUG("REQ depth length: "<length = i2d_PROXY_CERT_INFO_EXTENSION(pci, 0); - if (!(X509_EXTENSION_get_data(ext)->data = (unsigned char *)malloc(X509_EXTENSION_get_data(ext)->length+1))) { + X509_EXTENSION_get_data(ext.get())->length = i2d_PROXY_CERT_INFO_EXTENSION(pci.get(), 0); + if (!(X509_EXTENSION_get_data(ext.get())->data = (unsigned char *)malloc(X509_EXTENSION_get_data(ext.get())->length+1))) { PRINT("could not allocate data field for extension"); return -kErrPX_NoResources; } - unsigned char *pp = X509_EXTENSION_get_data(ext)->data; - if ((i2d_PROXY_CERT_INFO_EXTENSION(pci, &pp)) <= 0) { + unsigned char *pp = X509_EXTENSION_get_data(ext.get())->data; + if ((i2d_PROXY_CERT_INFO_EXTENSION(pci.get(), &pp)) <= 0) { PRINT("problem converting data for extension"); return -kErrPX_Error; } - PROXY_CERT_INFO_EXTENSION_free( pci ); + pci = nullptr; // Set extension name. ASN1_OBJECT *obj = OBJ_txt2obj(gsiProxyCertInfo_OID, 1); - if (!obj || X509_EXTENSION_set_object(ext, obj) != 1) { + if (!obj || X509_EXTENSION_set_object(ext.get(), obj) != 1) { PRINT("could not set extension name"); + ASN1_OBJECT_free( obj ); return -kErrPX_SetAttribute; } ASN1_OBJECT_free( obj ); + obj = 0; // flag as critical - if (X509_EXTENSION_set_critical(ext, 1) != 1) { + if (X509_EXTENSION_set_critical(ext.get(), 1) != 1) { PRINT("could not set extension critical flag"); return -kErrPX_SetAttribute; } - // Add the extension - if (X509_add_ext(xpo, ext, -1) == 0) { + // Add the extension (adds a copy of the extension) + if (X509_add_ext(xpo.get(), ext.get(), -1) == 0) { PRINT("could not add extension"); return -kErrPX_SetAttribute; } // // Sign the certificate - if (!(X509_sign(xpo, ekpi, EVP_sha1()))) { + if (!(X509_sign(xpo.get(), ekpi.get(), EVP_sha256()))) { PRINT("problems signing the certificate"); return -kErrPX_Signing; } - EVP_PKEY_free( ekpi ); // decrement reference counter - X509_EXTENSION_free( ext ); + ekpi = nullptr; + ext = nullptr; // Prepare outputs - *xcpo = new XrdCryptosslX509(xpo); + *xcpo = new XrdCryptosslX509(xpo.get()); - // Cleanup -#if OPENSSL_VERSION_NUMBER >= 0x10000000L - sk_X509_EXTENSION_free(xrisk); -#else /* OPENSSL */ - sk_free(xrisk); -#endif /* OPENSSL */ + // xpo resource is now owned by the *xcpo + xpo.release(); // We are done return 0; diff --git a/src/XrdCrypto/XrdCryptosslgsiAux.hh b/src/XrdCrypto/XrdCryptosslgsiAux.hh deleted file mode 100644 index aac394370e3..00000000000 --- a/src/XrdCrypto/XrdCryptosslgsiAux.hh +++ /dev/null @@ -1,102 +0,0 @@ -#ifndef __CRYPTO_SSLGSIAUX_H__ -#define __CRYPTO_SSLGSIAUX_H__ -/******************************************************************************/ -/* */ -/* X r d C r y p t o s s l g s i A u x . h h */ -/* */ -/* (c) 2005, G. Ganis / CERN */ -/* */ -/* This file is part of the XRootD software suite. */ -/* */ -/* XRootD is free software: you can redistribute it and/or modify it under */ -/* the terms of the GNU Lesser General Public License as published by the */ -/* Free Software Foundation, either version 3 of the License, or (at your */ -/* option) any later version. */ -/* */ -/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ -/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ -/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ -/* License for more details. */ -/* */ -/* You should have received a copy of the GNU Lesser General Public License */ -/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ -/* COPYING (GPL license). If not, see . */ -/* */ -/* The copyright holder's institutional names and contributor's names may not */ -/* be used to endorse or promote products derived from this software without */ -/* specific prior written permission of the institution or contributor. */ -/* */ -/******************************************************************************/ - -/* ************************************************************************** */ -/* */ -/* GSI utility functions */ -/* */ -/* ************************************************************************** */ -#include "XrdCrypto/XrdCryptosslgsiX509Chain.hh" -#include "XrdCrypto/XrdCryptoX509Req.hh" -#include "XrdCrypto/XrdCryptoRSA.hh" -#include "XrdOuc/XrdOucString.hh" - -// The OID of the extension -#define gsiProxyCertInfo_OLD_OID "1.3.6.1.4.1.3536.1.222" -#define gsiProxyCertInfo_OID "1.3.6.1.5.5.7.1.14" - -// -// Function to check presence of a proxyCertInfo and retrieve the path length -// constraint. Written following RFC3820 and examples in openssl-/crypto -// source code. Extracts the policy field but ignores it contents. -bool XrdSslgsiProxyCertInfo(const void *ext, int &pathlen, bool *haspolicy = 0); -void XrdSslgsiSetPathLenConstraint(void *ext, int pathlen); - -// -// Proxies -// -typedef struct { - int bits; // Number of bits in the RSA key [512] - int valid; // Duration validity in secs [43200 (12 hours)] - int depthlen; // Maximum depth of the path of proxy certificates - // that can signed by this proxy certificates - // [-1 (== unlimited)] -} XrdProxyOpt_t; -// -// Create proxy certificates -int XrdSslgsiX509CreateProxy(const char *, const char *, XrdProxyOpt_t *, - XrdCryptosslgsiX509Chain *, XrdCryptoRSA **, const char *); -// -// Create a proxy certificate request -int XrdSslgsiX509CreateProxyReq(XrdCryptoX509 *, - XrdCryptoX509Req **, XrdCryptoRSA **); -// -// Sign a proxy certificate request -int XrdSslgsiX509SignProxyReq(XrdCryptoX509 *, XrdCryptoRSA *, - XrdCryptoX509Req *, XrdCryptoX509 **); -// -// Dump extensions -int XrdSslgsiX509DumpExtensions(XrdCryptoX509 *); -// -// Get VOMS attributes, if any -int XrdSslgsiX509GetVOMSAttr(XrdCryptoX509 *, XrdOucString &); -// -// Check GSI 3 proxy info extension -int XrdSslgsiX509CheckProxy3(XrdCryptoX509 *, XrdOucString &); - -/******************************************************************************/ -/* E r r o r s i n P r o x y M a n i p u l a t i o n s */ -/******************************************************************************/ -#define kErrPX_Error 1 // Generic error condition -#define kErrPX_BadEECfile 2 // Absent or bad EEC cert or key file -#define kErrPX_BadEECkey 3 // Inconsistent EEC key -#define kErrPX_ExpiredEEC 4 // EEC is expired -#define kErrPX_NoResources 5 // Unable to create new objects -#define kErrPX_SetAttribute 6 // Unable to set a certificate attribute -#define kErrPX_SetPathDepth 7 // Unable to set path depth -#define kErrPX_Signing 8 // Problems signing -#define kErrPX_GenerateKey 9 // Problem generating the RSA key -#define kErrPX_ProxyFile 10 // Problem creating / updating proxy file -#define kErrPX_BadNames 11 // Names in certificates are bad -#define kErrPX_BadSerial 12 // Problems resolving serial number -#define kErrPX_BadExtension 13 // Problems with the extensions - -#endif - diff --git a/src/XrdCrypto/XrdCryptotest.cc b/src/XrdCrypto/XrdCryptotest.cc index 7c8f43ecfe3..fcdc4d0ee77 100644 --- a/src/XrdCrypto/XrdCryptotest.cc +++ b/src/XrdCrypto/XrdCryptotest.cc @@ -49,7 +49,7 @@ // // Globals -#define PRINT(x) {cerr <ExportPublic(RSApubexp,4096); - PRINT(outname<<": public export:"<GetPublen()); PRINT(outname<<": --------------------------------------------------- "); char RSApriexp[4096]; TestRSA_1->ExportPrivate(RSApriexp,4096); - PRINT(outname<<": private export:"<GetPrilen()); PRINT(outname<<": --------------------------------------------------- "); diff --git a/src/XrdDaemons.cmake b/src/XrdDaemons.cmake index dcde58d0bd3..0bfb7c20876 100644 --- a/src/XrdDaemons.cmake +++ b/src/XrdDaemons.cmake @@ -1,5 +1,4 @@ -include( XRootDCommon ) #------------------------------------------------------------------------------- # xrootd @@ -73,9 +72,3 @@ install( TARGETS xrootd cmsd RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ) - -install( - FILES - ${PROJECT_SOURCE_DIR}/docs/man/cmsd.8 - ${PROJECT_SOURCE_DIR}/docs/man/xrootd.8 - DESTINATION ${CMAKE_INSTALL_MANDIR}/man8 ) diff --git a/src/XrdEc/CMakeLists.txt b/src/XrdEc/CMakeLists.txt index 53754340c2c..ac56c33f7f6 100644 --- a/src/XrdEc/CMakeLists.txt +++ b/src/XrdEc/CMakeLists.txt @@ -1,4 +1,3 @@ -include( XRootDCommon ) include( ExternalProject ) #------------------------------------------------------------------------------- @@ -21,10 +20,8 @@ add_library( XrdEcReader.hh XrdEcReader.cc ) -target_link_libraries( - XrdEc - XrdCl -) +target_link_libraries(XrdEc PRIVATE XrdCl XrdUtils ${ISAL_LIBRARIES}) +target_include_directories(XrdEc PRIVATE ${ISAL_INCLUDE_DIRS}) set_target_properties( XrdEc @@ -43,15 +40,17 @@ install( #------------------------------------------------------------------------------ # Install private header files #------------------------------------------------------------------------------ -install( - FILES - XrdEcReader.hh - XrdEcObjCfg.hh - XrdEcStrmWriter.hh - XrdEcWrtBuff.hh - XrdEcThreadPool.hh - XrdEcUtilities.hh - XrdEcObjCfg.hh - XrdEcConfig.hh - XrdEcRedundancyProvider.hh - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xrootd/private/XrdEc ) +if (USE_SYSTEM_ISAL) + install( + FILES + XrdEcReader.hh + XrdEcObjCfg.hh + XrdEcStrmWriter.hh + XrdEcWrtBuff.hh + XrdEcThreadPool.hh + XrdEcUtilities.hh + XrdEcObjCfg.hh + XrdEcConfig.hh + XrdEcRedundancyProvider.hh + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xrootd/private/XrdEc ) +endif() diff --git a/src/XrdEc/XrdEcReader.cc b/src/XrdEc/XrdEcReader.cc index 62dcb046194..9ac55b5387e 100644 --- a/src/XrdEc/XrdEcReader.cc +++ b/src/XrdEc/XrdEcReader.cc @@ -929,7 +929,7 @@ namespace XrdEc } blockMap[blkid]->state[strpid] = block_t::Loading; - XrdCl::StatInfo* info; + XrdCl::StatInfo* info = nullptr; if(dataarchs[url]->Stat(objcfg.GetFileName(blkid, strpid), info).IsOK()) blockMap[blkid]->stripes[strpid].resize( info ->GetSize() ); diff --git a/src/XrdEc/XrdEcReader.hh b/src/XrdEc/XrdEcReader.hh index 3cfb75b4201..7b2de06480e 100644 --- a/src/XrdEc/XrdEcReader.hh +++ b/src/XrdEc/XrdEcReader.hh @@ -35,6 +35,7 @@ #include class MicroTest; +class XrdEcTests; namespace XrdEc { @@ -57,6 +58,7 @@ namespace XrdEc class Reader { friend class ::MicroTest; + friend class ::XrdEcTests; friend struct block_t; public: diff --git a/src/XrdEc/XrdEcStrmWriter.cc b/src/XrdEc/XrdEcStrmWriter.cc index 18da18a4fc1..4953a52e27f 100644 --- a/src/XrdEc/XrdEcStrmWriter.cc +++ b/src/XrdEc/XrdEcStrmWriter.cc @@ -211,7 +211,7 @@ namespace XrdEc writes.emplace_back( std::move( p ) ); } - XrdCl::Async( XrdCl::Parallel( writes ) >> [=]( XrdCl::XRootDStatus &st ){ global_status.report_wrt( st, blksize ); } ); + XrdCl::WaitFor( XrdCl::Parallel( writes ) >> [=]( XrdCl::XRootDStatus &st ){ global_status.report_wrt( st, blksize ); } ); } //--------------------------------------------------------------------------- diff --git a/src/XrdFfs.cmake b/src/XrdFfs.cmake index fea80acb392..d7ce6665850 100644 --- a/src/XrdFfs.cmake +++ b/src/XrdFfs.cmake @@ -1,5 +1,4 @@ -include( XRootDCommon ) #------------------------------------------------------------------------------- # Shared library version @@ -22,6 +21,7 @@ add_library( target_link_libraries( XrdFfs + PRIVATE XrdCl XrdPosix XrdUtils @@ -31,9 +31,7 @@ set_target_properties( XrdFfs PROPERTIES VERSION ${XRD_FFS_VERSION} - SOVERSION ${XRD_FFS_SOVERSION} - INTERFACE_LINK_LIBRARIES "" - LINK_INTERFACE_LIBRARIES "" ) + SOVERSION ${XRD_FFS_SOVERSION} ) #------------------------------------------------------------------------------- # xrootdfs @@ -49,6 +47,8 @@ if( BUILD_FUSE ) XrdPosix ${CMAKE_THREAD_LIBS_INIT} ${FUSE_LIBRARIES} ) + +target_include_directories(xrootdfs PRIVATE ${FUSE_INCLUDE_DIR}) endif() #------------------------------------------------------------------------------- @@ -62,9 +62,4 @@ if( BUILD_FUSE ) install( TARGETS xrootdfs RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) - - install( - FILES - ${PROJECT_SOURCE_DIR}/docs/man/xrootdfs.1 - DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 ) endif() diff --git a/src/XrdFfs/XrdFfsPosix.cc b/src/XrdFfs/XrdFfsPosix.cc index b6615db3693..20973446766 100644 --- a/src/XrdFfs/XrdFfsPosix.cc +++ b/src/XrdFfs/XrdFfsPosix.cc @@ -439,7 +439,7 @@ struct XrdFfsPosixX_readdirall_args { NULL in this case. Do we need some protection here? We are not in trouble so far - because FUSE's _getattr will test the existance of the dir + because FUSE's _getattr will test the existence of the dir so we know that at least one data server has the directory. */ void* XrdFfsPosix_x_readdirall(void* x) diff --git a/src/XrdFrc/XrdFrcReqAgent.cc b/src/XrdFrc/XrdFrcReqAgent.cc index 7434fc0a2b7..dd1267a7d63 100644 --- a/src/XrdFrc/XrdFrcReqAgent.cc +++ b/src/XrdFrc/XrdFrcReqAgent.cc @@ -125,7 +125,7 @@ int XrdFrcReqAgent::List(XrdFrcRequest::Item *Items, int Num) for (i = 0; i <= XrdFrcRequest::maxPrty; i++) {Offs = 0; while(rQueue[i]->List(myLfn, sizeof(myLfn), Offs, Items, Num)) - {cout <List(myLfn, sizeof(myLfn), Offs, Items, Num)) - {cout <Next = FSTab[0][n]; FSTab[0][n] = fsp; if (n > DYent) DYent = n; -//cerr <<"Add " <Age <<' ' <basePath() <Age <<' ' <basePath() <Next = FSTab[j][k]; else fsq = Insert(fsq, FSTab[j][k]); FSTab[j][k] = fsq; -//cerr <<"Bin " <Age <<' ' <basePath() <Age <<' ' <basePath() <Next)) SCent--; numEnt--; -//cerr <<"Oldest " <Age <<' ' <basePath() <Age <<' ' <basePath() <opaque?req->opaque->Env(envlen):""; - const char * resourcePlusOpaque = req->resourceplusopaque.c_str(); - headers["xrd-http-fullresource"] = resourcePlusOpaque != nullptr ? resourcePlusOpaque:""; + const char *p = nullptr; + if (req->opaque) + p = req->opaque->Env(envlen); + headers["xrd-http-query"] = p ? p:""; + p = req->resourceplusopaque.c_str(); + headers["xrd-http-fullresource"] = p ? p:""; headers["xrd-http-prot"] = prot->isHTTPS()?"https":"http"; // These fields usually identify the client that connected @@ -109,6 +112,10 @@ verb(req->requestverb), headers(req->allheaders) { clientgroups = prot->SecEntity.vorg; trim(clientgroups); } - + + // Get the packet marking handle and the client scitag from the XrdHttp layer + pmark = prot->pmarkHandle; + mSciTag = req->mScitag; + length = req->length; } diff --git a/src/XrdHttp/XrdHttpExtHandler.hh b/src/XrdHttp/XrdHttpExtHandler.hh index 7ddb2106cf4..06fd468ac07 100644 --- a/src/XrdHttp/XrdHttpExtHandler.hh +++ b/src/XrdHttp/XrdHttpExtHandler.hh @@ -36,6 +36,8 @@ #include #include +#include "XrdNet/XrdNetPMark.hh" + class XrdLink; class XrdSecEntity; class XrdHttpReq; @@ -55,6 +57,10 @@ public: std::string clientdn, clienthost, clientgroups; long long length; + XrdNetPMark * pmark; + + int mSciTag; + // Get full client identifier void GetClientID(std::string &clid); diff --git a/src/XrdHttp/XrdHttpProtocol.cc b/src/XrdHttp/XrdHttpProtocol.cc index feef9d6c447..0001b251fe3 100644 --- a/src/XrdHttp/XrdHttpProtocol.cc +++ b/src/XrdHttp/XrdHttpProtocol.cc @@ -109,7 +109,9 @@ XrdSecService *XrdHttpProtocol::CIA = 0; // Authentication Server int XrdHttpProtocol::m_bio_type = 0; // BIO type identifier for our custom BIO. BIO_METHOD *XrdHttpProtocol::m_bio_method = NULL; // BIO method constructor. char *XrdHttpProtocol::xrd_cslist = nullptr; +XrdNetPMark * XrdHttpProtocol::pmarkHandle = nullptr; XrdHttpChecksumHandler XrdHttpProtocol::cksumHandler = XrdHttpChecksumHandler(); +XrdHttpReadRangeHandler::Configuration XrdHttpProtocol::ReadRangeConfig; XrdSysTrace XrdHttpTrace("http"); @@ -185,7 +187,7 @@ int BIO_get_shutdown(BIO *bio) { XrdHttpProtocol::XrdHttpProtocol(bool imhttps) : XrdProtocol("HTTP protocol handler"), ProtLink(this), -SecEntity(""), CurrentReq(this) { +SecEntity(""), CurrentReq(this, ReadRangeConfig) { myBuff = 0; Addr_str = 0; Reset(); @@ -280,6 +282,7 @@ XrdProtocol *XrdHttpProtocol::Match(XrdLink *lp) { // that is is https without invoking TLS on the actual link. Eventually, // we should just use the link's TLS native implementation. // + hp->SecEntity.addrInfo = lp->AddrInfo(); XrdNetAddr *netP = const_cast(lp->NetAddr()); netP->SetDialect("https"); netP->SetTLS(true); @@ -579,9 +582,29 @@ int XrdHttpProtocol::Process(XrdLink *lp) // We ignore the argument here } else CurrentReq.reqstate++; + } else if (!DoneSetInfo && !CurrentReq.userAgent().empty()) { // DoingLogin is true, meaning the login finished. + std::string mon_info = "monitor info " + CurrentReq.userAgent(); + DoneSetInfo = true; + if (mon_info.size() >= 1024) { + TRACEI(ALL, "User agent string too long"); + } else if (!Bridge) { + TRACEI(ALL, "Internal logic error: Bridge is null after login"); + } else { + TRACEI(DEBUG, "Setting " << mon_info); + memset(&CurrentReq.xrdreq, 0, sizeof (ClientRequest)); + CurrentReq.xrdreq.set.requestid = htons(kXR_set); + CurrentReq.xrdreq.set.modifier = '\0'; + memset(CurrentReq.xrdreq.set.reserved, '\0', sizeof(CurrentReq.xrdreq.set.reserved)); + CurrentReq.xrdreq.set.dlen = htonl(mon_info.size()); + if (!Bridge->Run((char *) &CurrentReq.xrdreq, (char *) mon_info.c_str(), mon_info.size())) { + SendSimpleResp(500, nullptr, nullptr, "Could not set user agent.", 0, false); + return -1; + } + return 0; + } + } else { + DoingLogin = false; } - DoingLogin = false; - // Read the next request header, that is, read until a double CRLF is found @@ -950,6 +973,12 @@ int XrdHttpProtocol::Config(const char *ConfigFN, XrdOucEnv *myEnv) { char *var; int cfgFD, GoNo, NoGo = 0, ismine; + var = nullptr; + XrdOucEnv::Import("XRD_READV_LIMITS", var); + XrdHttpReadRangeHandler::Configure(eDest, var, ReadRangeConfig); + + pmarkHandle = (XrdNetPMark* ) myEnv->GetPtr("XrdNetPMark*"); + cksumHandler.configure(xrd_cslist); auto nonIanaChecksums = cksumHandler.getNonIANAConfiguredCksums(); if(nonIanaChecksums.size()) { @@ -1286,7 +1315,7 @@ int XrdHttpProtocol::getDataOneShot(int blen, bool wait) { // Check for buffer overflow first - maxread = min(blen, BuffAvailable()); + maxread = std::min(blen, BuffAvailable()); TRACE(DEBUG, "getDataOneShot BuffAvailable: " << BuffAvailable() << " maxread: " << maxread); if (!maxread) @@ -1298,7 +1327,7 @@ int XrdHttpProtocol::getDataOneShot(int blen, bool wait) { if (!wait) { int l = SSL_pending(ssl); if (l > 0) - sslavail = min(maxread, SSL_pending(ssl)); + sslavail = std::min(maxread, SSL_pending(ssl)); } if (sslavail < 0) { @@ -1471,10 +1500,10 @@ int XrdHttpProtocol::BuffgetData(int blen, char **data, bool wait) { // And now make available the data taken from the buffer. Note that the buffer // may be empty... if (myBuffStart <= myBuffEnd) { - rlen = min( (long) blen, (long)(myBuffEnd - myBuffStart) ); + rlen = std::min( (long) blen, (long)(myBuffEnd - myBuffStart) ); } else - rlen = min( (long) blen, (long)(myBuff->buff + myBuff->bsize - myBuffStart) ); + rlen = std::min( (long) blen, (long)(myBuff->buff + myBuff->bsize - myBuffStart) ); *data = myBuffStart; BuffConsume(rlen); @@ -1526,9 +1555,11 @@ int XrdHttpProtocol::StartSimpleResp(int code, const char *desc, const char *hea else if (code == 206) ss << "Partial Content"; else if (code == 302) ss << "Redirect"; else if (code == 307) ss << "Temporary Redirect"; + else if (code == 400) ss << "Bad Request"; else if (code == 403) ss << "Forbidden"; else if (code == 404) ss << "Not Found"; else if (code == 405) ss << "Method Not Allowed"; + else if (code == 416) ss << "Range Not Satisfiable"; else if (code == 500) ss << "Internal Server Error"; else ss << "Unknown"; } @@ -1578,10 +1609,11 @@ int XrdHttpProtocol::StartChunkedResp(int code, const char *desc, const char *he /******************************************************************************/ int XrdHttpProtocol::ChunkResp(const char *body, long long bodylen) { - if (ChunkRespHeader((bodylen <= 0) ? (body ? strlen(body) : 0) : bodylen)) + long long content_length = (bodylen <= 0) ? (body ? strlen(body) : 0) : bodylen; + if (ChunkRespHeader(content_length)) return -1; - if (body && SendData(body, bodylen)) + if (body && SendData(body, content_length)) return -1; return ChunkRespFooter(); @@ -1839,6 +1871,7 @@ void XrdHttpProtocol::Reset() { myBuffStart = myBuffEnd = 0; DoingLogin = false; + DoneSetInfo = false; ResumeBytes = 0; Resume = 0; diff --git a/src/XrdHttp/XrdHttpProtocol.hh b/src/XrdHttp/XrdHttpProtocol.hh index 0663dfd70c7..9d476480037 100644 --- a/src/XrdHttp/XrdHttpProtocol.hh +++ b/src/XrdHttp/XrdHttpProtocol.hh @@ -47,6 +47,8 @@ #include "Xrd/XrdProtocol.hh" #include "XrdOuc/XrdOucHash.hh" #include "XrdHttpChecksumHandler.hh" +#include "XrdHttpReadRangeHandler.hh" +#include "XrdNet/XrdNetPMark.hh" #include @@ -129,6 +131,9 @@ public: // XrdHttp checksum handling class static XrdHttpChecksumHandler cksumHandler; + /// configuration for the read range handler + static XrdHttpReadRangeHandler::Configuration ReadRangeConfig; + /// called via https bool isHTTPS() { return ishttps; } @@ -293,6 +298,9 @@ private: /// Tells that we are just logging in bool DoingLogin; + + /// Indicates whether we've attempted to send app info. + bool DoneSetInfo; /// Tells that we are just waiting to have N bytes in the buffer long ResumeBytes; @@ -429,5 +437,8 @@ protected: /// The list of checksums that were configured via the xrd.cksum parameter on the server config file static char * xrd_cslist; + + /// Packet marking handler pointer (assigned from the environment during the Config() call) + static XrdNetPMark * pmarkHandle; }; #endif diff --git a/src/XrdHttp/XrdHttpReadRangeHandler.cc b/src/XrdHttp/XrdHttpReadRangeHandler.cc new file mode 100644 index 00000000000..f6157bb6644 --- /dev/null +++ b/src/XrdHttp/XrdHttpReadRangeHandler.cc @@ -0,0 +1,658 @@ +//------------------------------------------------------------------------------ +// This file is part of XrdHTTP: A pragmatic implementation of the +// HTTP/WebDAV protocol for the Xrootd framework +// +// Copyright (c) 2013 by European Organization for Nuclear Research (CERN) +// Authors: Cedric Caffy , David Smith +// File Date: Aug 2023 +//------------------------------------------------------------------------------ +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +//------------------------------------------------------------------------------ + +#include "XProtocol/XPtypes.hh" +#include "XrdHttpReadRangeHandler.hh" +#include "XrdOuc/XrdOuca2x.hh" +#include "XrdOuc/XrdOucTUtils.hh" +#include "XrdOuc/XrdOucUtils.hh" + +#include +#include +#include +#include +#include +#include +#include + +//------------------------------------------------------------------------------ +//! static, class method: initialise a configuraiton object. parms is currently +//! only content of environment variable XRD_READV_LIMITS, to get the specific +//! kXR_readv limits. +//------------------------------------------------------------------------------ +int XrdHttpReadRangeHandler::Configure +( + XrdSysError &Eroute, + const char *const parms, + Configuration &cfg) +{ + if( !parms ) return 0; + + std::vector splitArgs; + XrdOucTUtils::splitString( splitArgs, parms, "," ); + if( splitArgs.size() < 2 ) return 0; + + //---------------------------------------------------------------------------- + // params is expected to be "," + //---------------------------------------------------------------------------- + std::string iorstr = splitArgs[0]; + std::string iovstr = splitArgs[1]; + XrdOucUtils::trim( iorstr ); + XrdOucUtils::trim( iovstr ); + + int val; + if( XrdOuca2x::a2i( Eroute, "Error reading specific value of readv_ior_max", + iorstr.c_str(), &val, 1, -1 ) ) + { + return -1; + } + + cfg.readv_ior_max = val; + if( XrdOuca2x::a2i( Eroute, "Error reading specific value of readv_iov_max", + iovstr.c_str(), &val, 1, -1 ) ) + { + return -1; + } + + cfg.readv_iov_max = val; + cfg.reqs_max = RREQ_MAXSIZE; + cfg.haveSizes = true; + + return 0; +} + +//------------------------------------------------------------------------------ +//! return the Error object +//------------------------------------------------------------------------------ +const XrdHttpReadRangeHandler::Error & XrdHttpReadRangeHandler::getError() const +{ + return error_; +} + +//------------------------------------------------------------------------------ +//! indicates when there were no valid Range head ranges supplied +//------------------------------------------------------------------------------ +bool XrdHttpReadRangeHandler::isFullFile() +{ + return rawUserRanges_.empty(); +} + +//------------------------------------------------------------------------------ +//! indicates a single range (implied whole file, or single range) or empty file +//------------------------------------------------------------------------------ +bool XrdHttpReadRangeHandler::isSingleRange() +{ + if( !rangesResolved_ ) + resolveRanges(); + + return( resolvedUserRanges_.size() <= 1 ); +} + +//------------------------------------------------------------------------------ +//! return resolved (i.e. obsolute start and end) byte ranges desired +//------------------------------------------------------------------------------ +const XrdHttpReadRangeHandler::UserRangeList &XrdHttpReadRangeHandler::ListResolvedRanges() +{ + static const UserRangeList emptyList; + + if( !rangesResolved_ ) + resolveRanges(); + + if( error_ ) + return emptyList; + + return resolvedUserRanges_; +} + +//------------------------------------------------------------------------------ +//! return XrdHttpIOList for sending to read or readv +//------------------------------------------------------------------------------ +const XrdHttpIOList &XrdHttpReadRangeHandler::NextReadList() +{ + static const XrdHttpIOList emptyList; + + if( !rangesResolved_ ) + resolveRanges(); + + if( error_ ) + return emptyList; + + if( !splitRange_.empty() ) + { + if( currSplitRangeIdx_ == 0 && currSplitRangeOff_ == 0 ) + { + //------------------------------------------------------------------------ + // Nothing read: Prevent scenario where data is expected but none is + // actually read E.g. Accessing files which return the results of a script + //------------------------------------------------------------------------ + error_.set( 500, "Stopping request because more data is expected " + "but no data has been read." ); + return emptyList; + } + + //-------------------------------------------------------------------------- + // we may have some unacknowledged portion of the last range; maybe due to a + // short read. so remove what was received and potentially reissue. + //-------------------------------------------------------------------------- + + trimSplit(); + if( !splitRange_.empty() ) + return splitRange_; + } + + if( splitRangeIdx_ >= resolvedUserRanges_.size() ) + return emptyList; + + splitRanges(); + + return splitRange_; +} + +//------------------------------------------------------------------------------ +//! Force handler to enter error state +//------------------------------------------------------------------------------ +void XrdHttpReadRangeHandler::NotifyError() +{ + if( error_ ) + return; + + error_.set( 500, "An error occured." ); +} + +//------------------------------------------------------------------------------ +//! Advance internal counters concerning received bytes +//------------------------------------------------------------------------------ +int XrdHttpReadRangeHandler::NotifyReadResult +( + const ssize_t ret, + const UserRange** const urp, + bool &start, + bool &allend +) +{ + if( error_ ) + return -1; + + if( ret == 0 ) + return 0; + + if( ret < 0 ) + { + error_.set( 500, "Range handler read failure." ); + return -1; + } + + if( !rangesResolved_ ) + { + error_.set( 500, "Range handler ranges not yet resolved." ); + return -1; + } + + if( splitRange_.empty() ) + { + error_.set( 500, "No ranges being read." ); + return -1; + } + + start = false; + allend = false; + + if( currSplitRangeIdx_ >= splitRange_.size() || + resolvedRangeIdx_ >= resolvedUserRanges_.size() ) + { + error_.set( 500, "Range handler index invalid." ); + return -1; + } + + if( urp ) + *urp = &resolvedUserRanges_[resolvedRangeIdx_]; + + if( resolvedRangeOff_ == 0 ) + start = true; + + const int clen = splitRange_[currSplitRangeIdx_].size; + + const off_t ulen = resolvedUserRanges_[resolvedRangeIdx_].end + - resolvedUserRanges_[resolvedRangeIdx_].start + 1; + + currSplitRangeOff_ += ret; + resolvedRangeOff_ += ret; + + if( currSplitRangeOff_ > clen || resolvedRangeOff_ > ulen ) + { + error_.set( 500, "Range handler read crossing chunk boundary." ); + return -1; + } + + if( currSplitRangeOff_ == clen ) + { + currSplitRangeOff_ = 0; + currSplitRangeIdx_++; + + if( currSplitRangeIdx_ >= splitRange_.size() ) + { + currSplitRangeIdx_ = 0; + splitRange_.clear(); + } + } + + if( resolvedRangeOff_ == ulen ) + { + resolvedRangeIdx_++; + resolvedRangeOff_ = 0; + if( resolvedRangeIdx_ >= resolvedUserRanges_.size() ) + allend = true; + } + + return 0; +} + +//------------------------------------------------------------------------------ +//! parse the line after a "Range: " http request header +//------------------------------------------------------------------------------ +void XrdHttpReadRangeHandler::ParseContentRange(const char* const line) +{ + char *str1, *saveptr1, *token; + + std::unique_ptr< char, decltype(std::free)* > + line_copy { strdup( line ), std::free }; + + //---------------------------------------------------------------------------- + // line_copy is argument of the Range header. + // + // e.g. "bytes=15-17,20-25" + // We skip the unit prefix (upto first '='). We don't + // enforce this prefix nor check what it is (e.g. 'bytes') + //---------------------------------------------------------------------------- + + str1 = line_copy.get(); + token = strchr(str1,'='); + if (token) str1 = token + 1; + + //---------------------------------------------------------------------------- + // break up the ranges and process each + //---------------------------------------------------------------------------- + + for( ; ; str1 = NULL ) + { + token = strtok_r( str1, " ,\n\r", &saveptr1 ); + if( token == NULL ) + break; + + if( !strlen(token) ) continue; + + const int rc = parseOneRange( token ); + if( rc ) + { + //------------------------------------------------------------------------ + // on error we ignore the whole range header + //------------------------------------------------------------------------ + rawUserRanges_.clear(); + return; + } + } +} + +//------------------------------------------------------------------------------ +//! resets this handler +//------------------------------------------------------------------------------ +void XrdHttpReadRangeHandler::reset() +{ + error_.reset(); + rawUserRanges_.clear(); + rawUserRanges_.shrink_to_fit(); + resolvedUserRanges_.clear(); + resolvedUserRanges_.shrink_to_fit(); + splitRange_.clear(); + splitRange_.shrink_to_fit(); + rangesResolved_ = false; + splitRangeIdx_ = 0; + splitRangeOff_ = 0; + currSplitRangeIdx_ = 0; + currSplitRangeOff_ = 0; + resolvedRangeIdx_ = 0; + resolvedRangeOff_ = 0; + filesize_ = 0; +} + +//------------------------------------------------------------------------------ +//! sets the filesize, used during resolving and issuing range requests +//------------------------------------------------------------------------------ +int XrdHttpReadRangeHandler::SetFilesize(const off_t fs) +{ + if( error_ ) + return -1; + + if( rangesResolved_ ) + { + error_.set( 500, "Filesize notified after ranges resolved." ); + return -1; + } + + filesize_ = fs; + return 0; +} + +//------------------------------------------------------------------------------ +//! private method: paring a single range from the header +//------------------------------------------------------------------------------ +int XrdHttpReadRangeHandler::parseOneRange(char* const str) +{ + UserRange ur; + char *sep; + + //---------------------------------------------------------------------------- + // expected input is an individual range, e.g. + // 5-6 + // 5- + // -2 + //---------------------------------------------------------------------------- + + sep = strchr( str, '-' ); + if( !sep ) + { + //-------------------------------------------------------------------------- + // Unexpected range format + //-------------------------------------------------------------------------- + return -1; + } + + *sep = '\0'; + if( rangeFig( str, ur.start_set, ur.start )<0 ) + { + //-------------------------------------------------------------------------- + // Error in range start + //-------------------------------------------------------------------------- + *sep = '-'; + return -1; + } + *sep = '-'; + if( rangeFig( sep+1, ur.end_set, ur.end )<0 ) + { + //-------------------------------------------------------------------------- + // Error in range end + //-------------------------------------------------------------------------- + return -1; + } + + if( !ur.start_set && !ur.end_set ) + { + //-------------------------------------------------------------------------- + // Unexpected range format + //-------------------------------------------------------------------------- + return -1; + } + + if( ur.start_set && ur.end_set && ur.start > ur.end ) + { + //-------------------------------------------------------------------------- + // Range start is after range end + //-------------------------------------------------------------------------- + return -1; + } + + if( !ur.start_set && ur.end_set && ur.end == 0 ) + { + //-------------------------------------------------------------------------- + // Request to return last 0 bytes of file + //-------------------------------------------------------------------------- + return -1; + } + + rawUserRanges_.push_back(ur); + return 0; +} + +//------------------------------------------------------------------------------ +//! private method: decode a decimal value from range header +//------------------------------------------------------------------------------ +int XrdHttpReadRangeHandler::rangeFig(const char* const s, bool &set, off_t &val) +{ + char *endptr = (char*)s; + errno = 0; + long long int v = strtoll( s, &endptr, 10 ); + if( (errno == ERANGE && (v == LONG_MAX || v == LONG_MIN)) + || (errno != 0 && errno != EINVAL && v == 0) ) + { + return -1; + } + if( *endptr != '\0' ) + { + return -1; + } + if( endptr == s ) + { + set = false; + } + else + { + set = true; + val = v; + } + return 0; +} + +//------------------------------------------------------------------------------ +//! private method: turn user supplied range into absolute range using filesize +//------------------------------------------------------------------------------ +void XrdHttpReadRangeHandler::resolveRanges() +{ + if( error_ ) + return; + + resolvedUserRanges_.clear(); + + for( const auto &rr: rawUserRanges_ ) + { + off_t start = 0; + off_t end = 0; + + if( rr.end_set ) + { + if( rr.start_set ) + { + //---------------------------------------------------------------------- + // end and start set + // e.g. 5-6 + //---------------------------------------------------------------------- + start = rr.start; + end = rr.end; + + //---------------------------------------------------------------------- + // skip ranges outside the file + //---------------------------------------------------------------------- + if( start >= filesize_ ) + continue; + + if( end >= filesize_ ) + { + end = filesize_ - 1; + } + } + else // !start + { + //---------------------------------------------------------------------- + // end is set but not start + // e.g. -5 + //---------------------------------------------------------------------- + if( rr.end == 0 ) + continue; + end = filesize_ -1; + if( rr.end > filesize_ ) + { + start = 0; + } + else + { + start = filesize_ - rr.end; + } + } + } + else // !end + { + //------------------------------------------------------------------------ + // end is not set + // e.g. 5- + //------------------------------------------------------------------------ + if( !rr.start_set ) continue; + if( rr.start >= filesize_ ) + continue; + start = rr.start; + end = filesize_ - 1; + } + resolvedUserRanges_.emplace_back( start, end ); + } + + if( rawUserRanges_.empty() && filesize_>0 ) + { + //-------------------------------------------------------------------------- + // special case: no ranges: speficied, return whole file + //-------------------------------------------------------------------------- + resolvedUserRanges_.emplace_back( 0, filesize_ - 1 ); + } + + if( !rawUserRanges_.empty() && resolvedUserRanges_.empty() ) + { + error_.set( 416, "None of the range-specifier values in the Range " + "request-header field overlap the current extent of the selected resource." ); + } + + rangesResolved_ = true; +} + +//------------------------------------------------------------------------------ +//! private method: proceed through the resolved ranges, splitting into ranges +//! suitable for read or readv. This method is called repeatedly until we've +//! gone though all the resolved ranges. +//------------------------------------------------------------------------------ +void XrdHttpReadRangeHandler::splitRanges() +{ + splitRange_.clear(); + currSplitRangeIdx_ = 0; + currSplitRangeOff_ = 0; + resolvedRangeIdx_ = splitRangeIdx_; + resolvedRangeOff_ = splitRangeOff_; + + //---------------------------------------------------------------------------- + // If we make a list of just one range XrdHttpReq will issue kXR_read, + // otherwise kXR_readv. + // + // If this is a full file read, or single user range, we'll fetch only one + // range at a time, so it is sent as a series of kXR_read requests. + // + // For multi range requests we pack a number of suitably sized ranges, thereby + // using kXR_readv. However, if there's a long user range we can we try to + // proceed by issuing single range requests and thereby using kXR_read. + // + // We don't merge user ranges in a single chunk as we always expect to be + // able to notify at boundaries with the output bools of NotifyReadResult. + //---------------------------------------------------------------------------- + + size_t maxch = vectorReadMaxChunks_; + size_t maxchs = vectorReadMaxChunkSize_; + if( isSingleRange() ) + { + maxchs = rRequestMaxBytes_; + maxch = 1; + } + + splitRange_.reserve( maxch ); + + //---------------------------------------------------------------------------- + // Start/continue splitting the resolvedUserRanges_ into a XrdHttpIOList. + //---------------------------------------------------------------------------- + + const size_t cs = resolvedUserRanges_.size(); + size_t nc = 0; + size_t rsr = rRequestMaxBytes_; + UserRange tmpur; + + while( ( splitRangeIdx_ < cs ) && ( rsr > 0 ) ) + { + //-------------------------------------------------------------------------- + // Check if we've readed the maximum number of allowed chunks. + //-------------------------------------------------------------------------- + if( nc >= maxch ) + break; + + if( !tmpur.start_set ) + { + tmpur = resolvedUserRanges_[splitRangeIdx_]; + tmpur.start += splitRangeOff_; + } + + const off_t l = tmpur.end - tmpur.start + 1; + size_t maxsize = std::min( rsr, maxchs ); + + //-------------------------------------------------------------------------- + // If we're starting a new set of chunks and we have enough data available + // in the current user range we allow a kXR_read of the max request size. + //-------------------------------------------------------------------------- + if( nc == 0 && l >= (off_t)rRequestMaxBytes_ ) + maxsize = rRequestMaxBytes_; + + if( l > (off_t)maxsize ) + { + splitRange_.emplace_back( nullptr, tmpur.start, maxsize ); + tmpur.start += maxsize; + splitRangeOff_ += maxsize; + rsr -= maxsize; + } + else + { + splitRange_.emplace_back( nullptr, tmpur.start, l ); + rsr -= l; + tmpur = UserRange(); + splitRangeOff_ = 0; + splitRangeIdx_++; + } + nc++; + } +} + +//------------------------------------------------------------------------------ +//! private method: remove partially received request +//------------------------------------------------------------------------------ +void XrdHttpReadRangeHandler::trimSplit() +{ + if( currSplitRangeIdx_ < splitRange_.size() ) + { + splitRange_.erase( splitRange_.begin(), + splitRange_.begin() + currSplitRangeIdx_ ); + } + else + splitRange_.clear(); + + if( splitRange_.size() > 0 ) + { + if( currSplitRangeOff_ < splitRange_[0].size ) + { + splitRange_[0].offset += currSplitRangeOff_; + splitRange_[0].size -= currSplitRangeOff_; + } + else + splitRange_.clear(); + } + + currSplitRangeIdx_ = 0; + currSplitRangeOff_ = 0; +} diff --git a/src/XrdHttp/XrdHttpReadRangeHandler.hh b/src/XrdHttp/XrdHttpReadRangeHandler.hh new file mode 100644 index 00000000000..b0b21b9bf32 --- /dev/null +++ b/src/XrdHttp/XrdHttpReadRangeHandler.hh @@ -0,0 +1,282 @@ +//------------------------------------------------------------------------------ +// This file is part of XrdHTTP: A pragmatic implementation of the +// HTTP/WebDAV protocol for the Xrootd framework +// +// Copyright (c) 2013 by European Organization for Nuclear Research (CERN) +// Authors: Cedric Caffy , David Smith +// File Date: Aug 2023 +//------------------------------------------------------------------------------ +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +//------------------------------------------------------------------------------ + +#ifndef XROOTD_XRDHTTPREADRANGEHANDLER_HH +#define XROOTD_XRDHTTPREADRANGEHANDLER_HH + +#include "XrdHttpUtils.hh" +#include +#include + + +/** + * Class responsible for parsing the HTTP Content-Range header + * coming from the client, generating appropriate read ranges + * for read or readv and tracking the responses to the requests. + */ +class XrdHttpReadRangeHandler { +public: + + /** + * These are defaults for: + * READV_MAXCHUNKS Max length of the XrdHttpIOList vector. + * READV_MAXCHUNKSIZE Max length of a XrdOucIOVec2 element. + * RREQ_MAXSIZE Max bytes to issue in a whole readv/read. + */ + static constexpr size_t READV_MAXCHUNKS = 512; + static constexpr size_t READV_MAXCHUNKSIZE = 512*1024; + static constexpr size_t RREQ_MAXSIZE = 8*1024*1024; + + /** + * Configuration can give specific values for the max chunk + * size, number of chunks and maximum overall request size, + * to override the defaults. + */ + struct Configuration { + Configuration() : haveSizes(false) { } + + Configuration(const size_t vectorReadMaxChunkSize, + const size_t vectorReadMaxChunks, + const size_t rRequestMaxBytes) : + haveSizes(true), readv_ior_max(vectorReadMaxChunkSize), + readv_iov_max(vectorReadMaxChunks), reqs_max(rRequestMaxBytes) { } + + + bool haveSizes; + size_t readv_ior_max; // max chunk size + size_t readv_iov_max; // max number of chunks + size_t reqs_max; // max bytes in read or readv + }; + + /** + * Error structure for storing error codes and message. + * operator bool() can be used to query if a value is set. + */ + struct Error { + bool errSet{false}; + int httpRetCode{0}; + std::string errMsg; + + explicit operator bool() const { return errSet; } + + void set(int rc, const std::string &m) + { httpRetCode = rc; errMsg = m; errSet = true; } + + void reset() { httpRetCode = 0; errMsg.clear(); errSet = false; } + }; + + /** + * Structure for recording or reporting user ranges. The can specify an + * unbounded range where eiter the start or end offset is not specified. + */ + struct UserRange { + bool start_set; + bool end_set; + off_t start; + off_t end; + + UserRange() : start_set(false), end_set(false), start(0), end(0) { } + + UserRange(off_t st, off_t en) : start_set(true), end_set(true), start(st), + end(en) { } + }; + + typedef std::vector UserRangeList; + + /** + * Constructor. + * Supplied with an Configuration object. The supplied object remains owned + * by the caller, but should remain valid throughout the lifetime of the + * ReadRangeHandler. + * + * @param @conf Configuration object. + */ + XrdHttpReadRangeHandler(const Configuration &conf) + { + rRequestMaxBytes_ = RREQ_MAXSIZE; + vectorReadMaxChunkSize_ = READV_MAXCHUNKSIZE; + vectorReadMaxChunks_ = READV_MAXCHUNKS; + + if( conf.haveSizes ) + { + vectorReadMaxChunkSize_ = conf.readv_ior_max; + vectorReadMaxChunks_ = conf.readv_iov_max; + rRequestMaxBytes_ = conf.reqs_max; + } + reset(); + } + + /** + * Parses a configuration into a Configuration object. + * @param @Eroute Error reporting object + * @param @parms Configuration string. + * @param @cfg an output Configuration object + * @return 0 for success, otherwise failure. + */ + static int Configure(XrdSysError &Eroute, const char *const parms, + Configuration &cfg); + + /** + * getter for the Error object. The object can be inspected with its operator + * bool() method to indicate an error has happened. Error code and message are + * available in other members of Error. + */ + const Error& getError() const; + + /** + * Indicates no valid Range header was given and thus the implication is that + * whole file is required. A range or ranges may be given that cover the whole + * file but that situation is not detected. + */ + bool isFullFile(); + + /** + * Incidcates whether there is a single range, either given by a Range header + * with single range or implied by having no Range header. + * Also returns true for an empty file, although there is no range of bytes. + * @return true if there is a single range. + */ + bool isSingleRange(); + + /** + * Returns a reference of the list of ranges. These are resolved, meaning that + * if there was no Range header, or it was in the form -N or N-, the file size + * is used to compute the actual range of bytes that are needed. The list + * remains owned by the handler and may be invalidated on reset(). + * @return List of ranges in a UserRangeList object. + * The returned list may be empty, i.e. for an empty file or if there + * is an error. Use getError() to see if there is an error. + */ + const UserRangeList &ListResolvedRanges(); + + /** + * Requests a XrdHttpIOList (vector of XrdOucIOVec2) that describes the next + * bytes that need to be fetched from a file. If there is more than one chunk + * it is size appropriately for a readv request, if there is one request it + * should be sent as a read request. Therefore the chunks do not necessarily + * correspond to the ranges the user requested. The caller issue the requests + * in the order provided and call NotifyReadResult with the ordered results. + * @return a reference to a XrdHttpIOList. The object remains owned by the + * handler. It may be invalided by a new call to NextReadList() or + * reset(). The returned list may be empty, which implies no more + * reads are needed One can use getError() to see if there is an + * error. + */ + const XrdHttpIOList &NextReadList(); + + /** + * Force the handler to enter error state. Sets a generic error message + * if there was not already an error. + */ + void NotifyError(); + + /** + * Notifies the handler about the arrival of bytes from a read or readv + * request. The handler tracks the progress of the arriving bytes against + * the bytes ranges the user requested. + * @param ret the number of bytes received + * @param urp a pointer to a pointer of a UserRange object. If urp is not + * nullptr, the pointer to a UserRange is returned that describes + * the current range associated with the received bytes. The + * handler retains ownership of the returned object. reset() of + * the handler invalidates the UserRange object. + * @param start is an output bool parameter that indicates whether the + * received bytes mark the start of a UserRange. + * @param allend is an output bool parameter that indicates whether the + * received bytes mark the end of all the UserRanges + * @return 0 upon success, -1 if an error happened. + * One needs to call the getError() method to return the error. + */ + int NotifyReadResult(const ssize_t ret, + const UserRange** const urp, + bool &start, + bool &allend); + + /** + * Parses the Content-Range header value and sets the ranges within the + * object. + * @param line the line under the format "bytes=0-19, 25-30" + * In case the parsing fails any partial results are cleared. There is no + * error notification as the rest of the request processing should continue + * in any case. + */ + void ParseContentRange(const char* const line); + + /** + * Resets the object state, ready for handling a new request. + */ + void reset(); + + /** + * Notifies of the current file size. This information is required for + * processing range requests that imply reading to the end or a certain + * position before the end of a file. It is also used to determine when read + * or readv need no longer be issued when reading the whole file. + * Can be called once or more, after reset() but before isSingleRange(), + * ListResolvedRanges() or NextReadList() methods. + * @param sz the size of the file + * @return 0 upon success, -1 if an error happened. + * One needs to call the getError() method to return the error. + */ + int SetFilesize(const off_t sz); + +private: + int parseOneRange(char* const str); + int rangeFig(const char* const s, bool &set, off_t &start); + void resolveRanges(); + void splitRanges(); + void trimSplit(); + + Error error_; + + UserRangeList rawUserRanges_; + + bool rangesResolved_; + + UserRangeList resolvedUserRanges_; + + XrdHttpIOList splitRange_; + + // the position in resolvedUserRanges_ corresponding to all the + // bytes notified via the NotifyReadResult() method + size_t resolvedRangeIdx_; + off_t resolvedRangeOff_; + + // position of the method splitRanges() in within resolvedUserRanges_ + // from where it split ranges into chunks for sending to read/readv + size_t splitRangeIdx_; + off_t splitRangeOff_; + + // the position in splitRange_ corresponding to all the + // bytes notified via the NotifyReadResult() method + size_t currSplitRangeIdx_; + int currSplitRangeOff_; + + off_t filesize_; + + size_t vectorReadMaxChunkSize_; + size_t vectorReadMaxChunks_; + size_t rRequestMaxBytes_; +}; + + +#endif //XROOTD_XRDHTTPREADRANGEHANDLER_HH diff --git a/src/XrdHttp/XrdHttpReq.cc b/src/XrdHttp/XrdHttpReq.cc index 7b7c2b18b4c..7d4beed9b75 100644 --- a/src/XrdHttp/XrdHttpReq.cc +++ b/src/XrdHttp/XrdHttpReq.cc @@ -86,7 +86,7 @@ std::string ISOdatetime(time_t t) { gmtime_r(&t, &t1); strftime(datebuf, 127, "%a, %d %b %Y %H:%M:%S GMT", &t1); - return (string) datebuf; + return (std::string) datebuf; } @@ -165,7 +165,10 @@ int XrdHttpReq::parseLine(char *line, int len) { } else if (!strcmp(key, "Host")) { parseHost(val); } else if (!strcmp(key, "Range")) { - parseContentRange(val); + // (rfc2616 14.35.1) says if Range header contains any range + // which is syntactically invalid the Range header should be ignored. + // Therefore no need for the range handler to report an error. + readRangeHandler.ParseContentRange(val); } else if (!strcmp(key, "Content-Length")) { length = atoll(val); @@ -184,26 +187,30 @@ int XrdHttpReq::parseLine(char *line, int len) { } else if (!strcmp(key, "Expect") && strstr(val, "100-continue")) { sendcontinue = true; - } else if (!strcasecmp(key, "Transfer-Encoding") && strstr(val, "chunked")) { - m_transfer_encoding_chunked = true; } else if (!strcasecmp(key, "TE") && strstr(val, "trailers")) { m_trailer_headers = true; + } else if (!strcasecmp(key, "Transfer-Encoding") && strstr(val, "chunked")) { + m_transfer_encoding_chunked = true; } else if (!strcasecmp(key, "X-Transfer-Status") && strstr(val, "true")) { + m_transfer_encoding_chunked = true; m_status_trailer = true; + } else if (!strcasecmp(key, "SciTag")) { + if(prot->pmarkHandle != nullptr) { + parseScitag(val); + } + } else if (!strcasecmp(key, "User-Agent")) { + m_user_agent = val; + trim(m_user_agent); } else { // Some headers need to be translated into "local" cgi info. - std::map< std:: string, std:: string > ::iterator it = prot->hdr2cgimap.find(key); + auto it = std::find_if(prot->hdr2cgimap.begin(), prot->hdr2cgimap.end(),[key](const auto & item) { + return !strcasecmp(key,item.first.c_str()); + }); if (it != prot->hdr2cgimap.end() && (opaque ? (0 == opaque->Get(it->second.c_str())) : true)) { - std:: string s; + std::string s; s.assign(val, line+len-val); trim(s); - - if (hdr2cgistr.length() > 0) { - hdr2cgistr.append("&"); - } - hdr2cgistr.append(it->second); - hdr2cgistr.append("="); - hdr2cgistr.append(s); + addCgi(it->second,s); } } @@ -220,87 +227,25 @@ int XrdHttpReq::parseHost(char *line) { return 0; } -int XrdHttpReq::parseContentRange(char *line) { - int j; - char *str1, *saveptr1, *token; - - - - for (j = 1, str1 = line;; j++, str1 = NULL) { - token = strtok_r(str1, " ,\n=", &saveptr1); - if (token == NULL) - break; - - //printf("%d: %s\n", j, token); - - if (!strlen(token)) continue; - - - parseRWOp(token); - - } - - return j; -} - -int XrdHttpReq::parseRWOp(char *str) { - ReadWriteOp o1; - int j; - char *saveptr2, *str2, *subtoken, *endptr; - bool ok = false; - - for (str2 = str, j = 0;; str2 = NULL, j++) { - subtoken = strtok_r(str2, "-", &saveptr2); - if (subtoken == NULL) - break; - - switch (j) { - case 0: - o1.bytestart = strtoll(subtoken, &endptr, 0); - if (!o1.bytestart && (endptr == subtoken)) o1.bytestart = -1; - break; - case 1: - o1.byteend = strtoll(subtoken, &endptr, 0); - if (!o1.byteend && (endptr == subtoken)) o1.byteend = -1; - ok = true; - break; - default: - // Malformed! - ok = false; - break; - } - - } - - - // This can be largely optimized - if (ok) { - - kXR_int32 len_ok = 0; - long long sz = o1.byteend - o1.bytestart + 1; - kXR_int32 newlen = sz; - - if (filesize > 0) - newlen = (kXR_int32) min(filesize - o1.bytestart, sz); - - rwOps.push_back(o1); - - while (len_ok < newlen) { - ReadWriteOp nfo; - int len = min(newlen - len_ok, READV_MAXCHUNKSIZE); - - nfo.bytestart = o1.bytestart + len_ok; - nfo.byteend = nfo.bytestart + len - 1; - len_ok += len; - rwOps_split.push_back(nfo); +void XrdHttpReq::parseScitag(const std::string & val) { + // The scitag header has been populated and the packet marking was configured, the scitag will either be equal to 0 + // or to the value passed by the client + mScitag = 0; + std::string scitagS = val; + trim(scitagS); + if(scitagS.size()) { + if(scitagS[0] != '-') { + try { + mScitag = std::stoi(scitagS.c_str(), nullptr, 10); + if (mScitag > XrdNetPMark::maxTotID || mScitag < XrdNetPMark::minTotID) { + mScitag = 0; + } + } catch (...) { + //Nothing to do, scitag = 0 by default + } } - length += len_ok; - - } - - - return j; + addCgi("scitag.flow", std::to_string(mScitag)); } int XrdHttpReq::parseFirstLine(char *line, int len) { @@ -435,29 +380,23 @@ void XrdHttpReq::clientUnMarshallReadAheadList(int nitems) { } } -int XrdHttpReq::ReqReadV() { +int XrdHttpReq::ReqReadV(const XrdHttpIOList &cl) { - kXR_int64 total_len = 0; - rwOpPartialDone = 0; // Now we build the protocol-ready read ahead list // and also put the correct placeholders inside the cache - int n = rwOps_split.size(); - if (!ralist) ralist = (readahead_list *) malloc(n * sizeof (readahead_list)); + int n = cl.size(); + ralist.clear(); + ralist.reserve(n); int j = 0; - for (int i = 0; i < n; i++) { - - // We can suppose that we know the length of the file - // Hence we can sort out requests that are out of boundary or trim them - if (rwOps_split[i].bytestart > filesize) continue; - if (rwOps_split[i].byteend > filesize - 1) rwOps_split[i].byteend = filesize - 1; + for (const auto &c: cl) { + ralist.emplace_back(); + auto &ra = ralist.back(); + memcpy(&ra.fhandle, this->fhandle, 4); - memcpy(&(ralist[j].fhandle), this->fhandle, 4); - - ralist[j].offset = rwOps_split[i].bytestart; - ralist[j].rlen = rwOps_split[i].byteend - rwOps_split[i].bytestart + 1; - total_len += ralist[j].rlen; + ra.offset = c.offset; + ra.rlen = c.size; j++; } @@ -479,7 +418,7 @@ int XrdHttpReq::ReqReadV() { } std::string XrdHttpReq::buildPartialHdr(long long bytestart, long long byteend, long long fsz, char *token) { - ostringstream s; + std::ostringstream s; s << "\r\n--" << token << "\r\n"; s << "Content-type: text/plain; charset=UTF-8\r\n"; @@ -489,7 +428,7 @@ std::string XrdHttpReq::buildPartialHdr(long long bytestart, long long byteend, } std::string XrdHttpReq::buildPartialHdrEnd(char *token) { - ostringstream s; + std::ostringstream s; s << "\r\n--" << token << "--\r\n"; @@ -522,11 +461,21 @@ int XrdHttpReq::File(XrdXrootd::Bridge::Context &info, //!< the result context int dlen //!< byte count ) { + // sendfile about to be sent by bridge for fetching data for GET: + // no https, no chunked+trailer, no multirange + //prot->SendSimpleResp(200, NULL, NULL, NULL, dlen); int rc = info.Send(0, 0, 0, 0); TRACE(REQ, " XrdHttpReq::File dlen:" << dlen << " send rc:" << rc); - if (rc) return false; - writtenbytes += dlen; + bool start, finish; + // short read will be classed as error + if (rc) { + readRangeHandler.NotifyError(); + return false; + } + + if (readRangeHandler.NotifyReadResult(dlen, nullptr, start, finish) < 0) + return false; return true; @@ -537,7 +486,8 @@ bool XrdHttpReq::Done(XrdXrootd::Bridge::Context & info) { TRACE(REQ, " XrdHttpReq::Done"); xrdresp = kXR_ok; - + + this->iovN = 0; int r = PostProcessHTTPReq(true); // Beware, we don't have to reset() if the result is 0 @@ -825,6 +775,15 @@ void XrdHttpReq::sanitizeResourcePfx() { } } +void XrdHttpReq::addCgi(const std::string &key, const std::string &value) { + if (hdr2cgistr.length() > 0) { + hdr2cgistr.append("&"); + } + hdr2cgistr.append(key); + hdr2cgistr.append("="); + hdr2cgistr.append(value); +} + // Parse a resource line: // - sanitize @@ -927,7 +886,14 @@ void XrdHttpReq::mapXrdErrorToHttpStatus() { httpStatusCode = 409; httpStatusText = "Resource is a directory"; break; case kXR_ItExists: - httpStatusCode = 409; httpStatusText = "File already exists"; + if(request != ReqType::rtDELETE) { + httpStatusCode = 409; httpStatusText = "File already exists"; + } else { + // In the case the XRootD layer returns a kXR_ItExists after a deletion + // was submitted, we return a 405 status code with the error message set by + // the XRootD layer + httpStatusCode = 405; + } break; case kXR_InvalidRequest: httpStatusCode = 405; httpStatusText = "Method is not allowed"; @@ -1107,6 +1073,11 @@ int XrdHttpReq::ProcessHTTPReq() { } + // The reqstate parameter basically moves us through a simple state machine. + // - 0: Perform a stat on the resource + // - 1: Perform a checksum request on the resource (only if requested in header; otherwise skipped) + // - 2: Perform an open request (dirlist as appropriate). + // - 3+: Reads from file; if at end, perform a close. switch (reqstate) { case 0: // Stat() @@ -1119,7 +1090,36 @@ int XrdHttpReq::ProcessHTTPReq() { } return 0; - case 1: // Open() or dirlist + case 1: // Checksum request + if (!(fileflags & kXR_isDir) && !m_req_digest.empty()) { + // In this case, the Want-Digest header was set. + bool has_opaque = strchr(resourceplusopaque.c_str(), '?'); + // Note that doChksum requires that the memory stays alive until the callback is invoked. + m_req_cksum = prot->cksumHandler.getChecksumToRun(m_req_digest); + if(!m_req_cksum) { + // No HTTP IANA checksums have been configured by the server admin, return a "METHOD_NOT_ALLOWED" error + prot->SendSimpleResp(403, NULL, NULL, (char *) "No HTTP-IANA compatible checksums have been configured.", 0, false); + return -1; + } + m_resource_with_digest = resourceplusopaque; + if (has_opaque) { + m_resource_with_digest += "&cks.type="; + m_resource_with_digest += m_req_cksum->getXRootDConfigDigestName().c_str(); + } else { + m_resource_with_digest += "?cks.type="; + m_resource_with_digest += m_req_cksum->getXRootDConfigDigestName().c_str(); + } + if (prot->doChksum(m_resource_with_digest) < 0) { + prot->SendSimpleResp(500, NULL, NULL, (char *) "Failed to start internal checksum request to satisfy Want-Digest header.", 0, false); + return -1; + } + return 0; + } else { + TRACEI(DEBUG, "No checksum requested; skipping to request state 2"); + reqstate += 1; + } + // fallthrough + case 2: // Open() or dirlist { if (!prot->Bridge) { @@ -1149,7 +1149,7 @@ int XrdHttpReq::ProcessHTTPReq() { } - string res; + std::string res; res = resourceplusopaque.c_str(); //res += "?xrd.dirstat=1"; @@ -1168,29 +1168,6 @@ int XrdHttpReq::ProcessHTTPReq() { // We don't want to be invoked again after this request is finished return 1; - } else if (!m_req_digest.empty()) { - // In this case, the Want-Digest header was set. - bool has_opaque = strchr(resourceplusopaque.c_str(), '?'); - // Note that doChksum requires that the memory stays alive until the callback is invoked. - m_req_cksum = prot->cksumHandler.getChecksumToRun(m_req_digest); - if(!m_req_cksum) { - // No HTTP IANA checksums have been configured by the server admin, return a "METHOD_NOT_ALLOWED" error - prot->SendSimpleResp(403, NULL, NULL, (char *) "No HTTP-IANA compatible checksums have been configured.", 0, false); - return -1; - } - m_resource_with_digest = resourceplusopaque; - if (has_opaque) { - m_resource_with_digest += "&cks.type="; - m_resource_with_digest += m_req_cksum->getXRootDConfigDigestName().c_str(); - } else { - m_resource_with_digest += "?cks.type="; - m_resource_with_digest += m_req_cksum->getXRootDConfigDigestName().c_str(); - } - if (prot->doChksum(m_resource_with_digest) < 0) { - prot->SendSimpleResp(500, NULL, NULL, (char *) "Failed to start internal checksum request to satisfy Want-Digest header.", 0, false); - return -1; - } - return 0; } else { @@ -1216,40 +1193,18 @@ int XrdHttpReq::ProcessHTTPReq() { } - } - case 2: // Open() in the case the user also requested a checksum. - { - if (!m_req_digest.empty()) { - // --------- OPEN - memset(&xrdreq, 0, sizeof (ClientRequest)); - xrdreq.open.requestid = htons(kXR_open); - l = resourceplusopaque.length() + 1; - xrdreq.open.dlen = htonl(l); - xrdreq.open.mode = 0; - xrdreq.open.options = htons(kXR_retstat | kXR_open_read); - - if (!prot->Bridge->Run((char *) &xrdreq, (char *) resourceplusopaque.c_str(), l)) { - prot->SendSimpleResp(404, NULL, NULL, (char *) "Could not run request.", 0, false); - return -1; - } - - // Prepare to chunk up the request - writtenbytes = 0; - - // We want to be invoked again after this request is finished - return 0; - } } // fallthrough - default: // Read() or Close() + default: // Read() or Close(); reqstate is 3+ { - if ( ((reqstate == 3 || (!m_req_digest.empty() && (reqstate == 4))) && (rwOps.size() > 1)) || - (writtenbytes >= length) ) { + const XrdHttpIOList &readChunkList = readRangeHandler.NextReadList(); - // Close() if this was a readv or we have finished, otherwise read the next chunk + // Close() if we have finished, otherwise read the next chunk - // --------- CLOSE + // --------- CLOSE + if ( readChunkList.empty() ) + { memset(&xrdreq, 0, sizeof (ClientRequest)); xrdreq.close.requestid = htons(kXR_close); @@ -1261,12 +1216,14 @@ int XrdHttpReq::ProcessHTTPReq() { } // We have finished + readClosing = true; return 1; } - - if (rwOps.size() <= 1) { - // No chunks or one chunk... Request the whole file or single read + // --------- READ or READV + + if ( readChunkList.size() == 1 ) { + // Use a read request for single range long l; long long offs; @@ -1277,21 +1234,17 @@ int XrdHttpReq::ProcessHTTPReq() { memcpy(xrdreq.read.fhandle, fhandle, 4); xrdreq.read.dlen = 0; - if (rwOps.size() == 0) { - l = (long)min(filesize-writtenbytes, (long long)1024*1024); - offs = writtenbytes; - xrdreq.read.offset = htonll(writtenbytes); - xrdreq.read.rlen = htonl(l); - } else { - l = min(rwOps[0].byteend - rwOps[0].bytestart + 1 - writtenbytes, (long long)1024*1024); - offs = rwOps[0].bytestart + writtenbytes; - xrdreq.read.offset = htonll(offs); - xrdreq.read.rlen = htonl(l); - } + offs = readChunkList[0].offset; + l = readChunkList[0].size; + + xrdreq.read.offset = htonll(offs); + xrdreq.read.rlen = htonl(l); - // If we are using HTTPS or if the client requested trailers, disable sendfile - // (in the latter case, the chunked encoding prevents sendfile usage) - if (prot->ishttps || (m_transfer_encoding_chunked && m_trailer_headers)) { + // If we are using HTTPS or if the client requested trailers, or if the + // read concerns a multirange reponse, disable sendfile + // (in the latter two cases, the extra framing is only done in PostProcessHTTPReq) + if (prot->ishttps || (m_transfer_encoding_chunked && m_trailer_headers) || + !readRangeHandler.isSingleRange()) { if (!prot->Bridge->setSF((kXR_char *) fhandle, false)) { TRACE(REQ, " XrdBridge::SetSF(false) failed."); @@ -1324,11 +1277,11 @@ int XrdHttpReq::ProcessHTTPReq() { return -1; } } else { - // More than one chunk to read... use readv + // --------- READV - length = ReqReadV(); + length = ReqReadV(readChunkList); - if (!prot->Bridge->Run((char *) &xrdreq, (char *) ralist, length)) { + if (!prot->Bridge->Run((char *) &xrdreq, (char *) &ralist[0], length)) { prot->SendSimpleResp(404, NULL, NULL, (char *) "Could not run read request.", 0, false); return -1; } @@ -1337,12 +1290,12 @@ int XrdHttpReq::ProcessHTTPReq() { // We want to be invoked again after this request is finished return 0; - } + } // case 3+ - } + } // switch (reqstate) - } + } // case XrdHttpReq::rtGET case XrdHttpReq::rtPUT: { @@ -1408,14 +1361,24 @@ int XrdHttpReq::ProcessHTTPReq() { // Parse out the next chunk size. long long idx = 0; bool found_newline = false; - for (; idx < prot->BuffAvailable(); idx++) { + // Set a maximum size of chunk we will allow + // Nginx sets this to "NGX_MAX_OFF_T_VALUE", which is 9223372036854775807 (a some crazy number) + // We set it to 1TB, which is 1099511627776 + // This is to prevent a malicious client from sending a very large chunk size + // or a malformed chunk request. + // 1TB in base-16 is 0x40000000000, so only allow 11 characters, plus the CRLF + long long max_chunk_size_chars = std::min(static_cast(prot->BuffUsed()), static_cast(13)); + for (; idx < max_chunk_size_chars; idx++) { if (prot->myBuffStart[idx] == '\n') { found_newline = true; break; } } - if ((idx == 0) || prot->myBuffStart[idx-1] != '\r') { + // If we found a new line, but it is the first character in the buffer (no chunk length) + // or if the previous character is not a CR. + if (found_newline && ((idx == 0) || prot->myBuffStart[idx-1] != '\r')) { prot->SendSimpleResp(400, NULL, NULL, (char *)"Invalid chunked encoding", 0, false); + TRACE(REQ, "XrdHTTP PUT: Sending invalid chunk encoding. Start of chunk should have had a length, followed by a CRLF."); return -1; } if (found_newline) { @@ -1425,6 +1388,7 @@ int XrdHttpReq::ProcessHTTPReq() { // Chunk sizes can be followed by trailer information or CRLF if (*endptr != ';' && *endptr != '\r') { prot->SendSimpleResp(400, NULL, NULL, (char *)"Invalid chunked encoding", 0, false); + TRACE(REQ, "XrdHTTP PUT: Sending invalid chunk encoding. Chunk size was not followed by a ';' or CR." << __LINE__); return -1; } m_current_chunk_size = chunk_contents; @@ -1447,13 +1411,13 @@ int XrdHttpReq::ProcessHTTPReq() { memcpy(xrdreq.write.fhandle, fhandle, 4); long long chunk_bytes_remaining = m_current_chunk_size - m_current_chunk_offset; - long long bytes_to_write = min(static_cast(prot->BuffUsed()), + long long bytes_to_write = std::min(static_cast(prot->BuffUsed()), chunk_bytes_remaining); xrdreq.write.offset = htonll(writtenbytes); xrdreq.write.dlen = htonl(bytes_to_write); - TRACEI(REQ, "Writing chunk of size " << bytes_to_write << " starting with '" << *(prot->myBuffStart) << "'"); + TRACEI(REQ, "XrdHTTP PUT: Writing chunk of size " << bytes_to_write << " starting with '" << *(prot->myBuffStart) << "'" << " with " << chunk_bytes_remaining << " bytes remaining in the chunk"); if (!prot->Bridge->Run((char *) &xrdreq, prot->myBuffStart, bytes_to_write)) { prot->SendSimpleResp(500, NULL, NULL, (char *) "Could not run write request.", 0, false); return -1; @@ -1470,7 +1434,7 @@ int XrdHttpReq::ProcessHTTPReq() { xrdreq.write.requestid = htons(kXR_write); memcpy(xrdreq.write.fhandle, fhandle, 4); - long long bytes_to_read = min(static_cast(prot->BuffUsed()), + long long bytes_to_read = std::min(static_cast(prot->BuffUsed()), length - writtenbytes); xrdreq.write.offset = htonll(writtenbytes); @@ -1534,7 +1498,7 @@ int XrdHttpReq::ProcessHTTPReq() { // --------- STAT is always the first step memset(&xrdreq, 0, sizeof (ClientRequest)); xrdreq.stat.requestid = htons(kXR_stat); - string s = resourceplusopaque.c_str(); + std::string s = resourceplusopaque.c_str(); l = resourceplusopaque.length() + 1; @@ -1555,7 +1519,7 @@ int XrdHttpReq::ProcessHTTPReq() { memset(&xrdreq, 0, sizeof (ClientRequest)); xrdreq.rmdir.requestid = htons(kXR_rmdir); - string s = resourceplusopaque.c_str(); + std::string s = resourceplusopaque.c_str(); l = s.length() + 1; xrdreq.rmdir.dlen = htonl(l); @@ -1569,7 +1533,7 @@ int XrdHttpReq::ProcessHTTPReq() { memset(&xrdreq, 0, sizeof (ClientRequest)); xrdreq.rm.requestid = htons(kXR_rm); - string s = resourceplusopaque.c_str(); + std::string s = resourceplusopaque.c_str(); l = s.length() + 1; xrdreq.rm.dlen = htonl(l); @@ -1628,7 +1592,7 @@ int XrdHttpReq::ProcessHTTPReq() { // --------- STAT is always the first step memset(&xrdreq, 0, sizeof (ClientRequest)); xrdreq.stat.requestid = htons(kXR_stat); - string s = resourceplusopaque.c_str(); + std::string s = resourceplusopaque.c_str(); l = resourceplusopaque.length() + 1; @@ -1659,7 +1623,7 @@ int XrdHttpReq::ProcessHTTPReq() { memset(&xrdreq, 0, sizeof (ClientRequest)); xrdreq.dirlist.requestid = htons(kXR_dirlist); - string s = resourceplusopaque.c_str(); + std::string s = resourceplusopaque.c_str(); xrdreq.dirlist.options[0] = kXR_dstat; //s += "?xrd.dirstat=1"; @@ -1686,7 +1650,7 @@ int XrdHttpReq::ProcessHTTPReq() { memset(&xrdreq, 0, sizeof (ClientRequest)); xrdreq.mkdir.requestid = htons(kXR_mkdir); - string s = resourceplusopaque.c_str(); + std::string s = resourceplusopaque.c_str(); xrdreq.mkdir.options[0] = (kXR_char) kXR_mkdirpath; l = s.length() + 1; @@ -1707,7 +1671,7 @@ int XrdHttpReq::ProcessHTTPReq() { memset(&xrdreq, 0, sizeof (ClientRequest)); xrdreq.mv.requestid = htons(kXR_mv); - string s = resourceplusopaque.c_str(); + std::string s = resourceplusopaque.c_str(); s += " "; char buf[256]; @@ -1799,7 +1763,7 @@ XrdHttpReq::PostProcessChecksum(std::string &digest_header) { if (convert_to_base64) {free(digest_value);} return 0; } else { - prot->SendSimpleResp(500, NULL, NULL, "Underlying filesystem failed to calculate checksum.", 0, false); + prot->SendSimpleResp(httpStatusCode, NULL, NULL, httpStatusText.c_str(), httpStatusText.length(), false); return -1; } } @@ -1809,9 +1773,18 @@ XrdHttpReq::PostProcessChecksum(std::string &digest_header) { int XrdHttpReq::PostProcessHTTPReq(bool final_) { - TRACEI(REQ, "PostProcessHTTPReq req: " << request << " reqstate: " << reqstate); + TRACEI(REQ, "PostProcessHTTPReq req: " << request << " reqstate: " << reqstate << " final_:" << final_); mapXrdErrorToHttpStatus(); + if(xrdreq.set.requestid == htons(kXR_set)) { + // We have set the user agent, if it fails we return a 500 error, otherwise the callback is successful --> we continue + if(xrdresp != kXR_ok) { + prot->SendSimpleResp(500, nullptr, nullptr, "Could not set user agent.", 0, false); + return -1; + } + return 0; + } + switch (request) { case XrdHttpReq::rtUnknown: { @@ -1953,7 +1926,7 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { if (e.path.length() && (e.path != ".") && (e.path != "..")) { // The entry is filled. file1.txt - string p = "" + std::string p = "" ""; if (e.flags & kXR_isDir) p += "d"; @@ -2080,10 +2053,15 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { } - } else { - - + } // end handling of dirlist + else + { // begin handling of open-read-close + // To duplicate the state diagram from the rtGET request state + // - 0: Perform a stat on the resource + // - 1: Perform a checksum request on the resource (only if requested in header; otherwise skipped) + // - 2: Perform an open request (dirlist as appropriate). + // - 3+: Reads from file; if at end, perform a close. switch (reqstate) { case 0: //stat { @@ -2106,6 +2084,8 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { &fileflags, &filemodtime); + readRangeHandler.SetFilesize(filesize); + // We will default the response size specified by the headers; if that // wasn't given, use the file size. if (!length) { @@ -2129,21 +2109,14 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { // We are here in the case of a negative response in a non-manager return 0; + } // end stat + case 1: // checksum was requested and now we have its response. + { + return PostProcessChecksum(m_digest_header); } - case 1: // open - case 2: // open when digest was requested + case 2: // open { - - if (reqstate == 1 && !m_req_digest.empty()) { // We requested a checksum and now have its response. - int response = PostProcessChecksum(m_digest_header); - if (-1 == response) { - return -1; - } - return 0; - } else if (((reqstate == 2 && !m_req_digest.empty()) || - (reqstate == 1 && m_req_digest.empty())) - && (xrdresp == kXR_ok)) { - + if (xrdresp == kXR_ok) { getfhandle(); @@ -2160,6 +2133,8 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { &fileflags, &filemodtime); + readRangeHandler.SetFilesize(filesize); + // As above: if the client specified a response size, we use that. // Otherwise, utilize the filesize if (!length) { @@ -2183,87 +2158,96 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { responseHeader += std::string("Age: ") + std::to_string(object_age < 0 ? 0 : object_age); } - if (rwOps.size() == 0) { + const XrdHttpReadRangeHandler::UserRangeList &uranges = readRangeHandler.ListResolvedRanges(); + if (uranges.empty() && readRangeHandler.getError()) { + prot->SendSimpleResp(readRangeHandler.getError().httpRetCode, NULL, NULL, readRangeHandler.getError().errMsg.c_str(),0,false); + return -1; + } + + if (readRangeHandler.isFullFile()) { // Full file. if (m_transfer_encoding_chunked && m_trailer_headers) { - prot->StartChunkedResp(200, NULL, responseHeader.empty() ? NULL : responseHeader.c_str(), filesize, keepalive); + prot->StartChunkedResp(200, NULL, responseHeader.empty() ? NULL : responseHeader.c_str(), -1, keepalive); } else { prot->SendSimpleResp(200, NULL, responseHeader.empty() ? NULL : responseHeader.c_str(), NULL, filesize, keepalive); } return 0; - } else - if (rwOps.size() == 1) { - // Only one read to perform - if (rwOps[0].byteend < 0) // The requested range was along the lines of "Range: 1234-", meaning we need to fill in the end - rwOps[0].byteend = filesize - 1; - int cnt = (rwOps[0].byteend - rwOps[0].bytestart + 1); + } + + if (readRangeHandler.isSingleRange()) { + // Possibly with zero sized file but should have been included + // in the FullFile case above + if (uranges.size() != 1) + return -1; + + // Only one range to return to the user char buf[64]; - + const off_t cnt = uranges[0].end - uranges[0].start + 1; + XrdOucString s = "Content-Range: bytes "; - sprintf(buf, "%lld-%lld/%lld", rwOps[0].bytestart, rwOps[0].byteend, filesize); + sprintf(buf, "%lld-%lld/%lld", (long long int)uranges[0].start, (long long int)uranges[0].end, filesize); s += buf; if (!responseHeader.empty()) { s += "\r\n"; s += responseHeader.c_str(); } - prot->SendSimpleResp(206, NULL, (char *)s.c_str(), NULL, cnt, keepalive); - return 0; - } else - if (rwOps.size() > 1) { - // Multiple reads to perform, compose and send the header - int cnt = 0; - for (size_t i = 0; i < rwOps.size(); i++) { - - if (rwOps[i].bytestart > filesize) continue; - if (rwOps[i].byteend > filesize - 1) - rwOps[i].byteend = filesize - 1; - - cnt += (rwOps[i].byteend - rwOps[i].bytestart + 1); - - cnt += buildPartialHdr(rwOps[i].bytestart, - rwOps[i].byteend, - filesize, - (char *) "123456").size(); - } - cnt += buildPartialHdrEnd((char *) "123456").size(); - std::string header = "Content-Type: multipart/byteranges; boundary=123456"; - if (!m_digest_header.empty()) { - header += "\n"; - header += m_digest_header; + if (m_transfer_encoding_chunked && m_trailer_headers) { + prot->StartChunkedResp(206, NULL, (char *)s.c_str(), -1, keepalive); + } else { + prot->SendSimpleResp(206, NULL, (char *)s.c_str(), NULL, cnt, keepalive); } - - prot->SendSimpleResp(206, NULL, header.c_str(), NULL, cnt, keepalive); return 0; } + // Multiple reads to perform, compose and send the header + off_t cnt = 0; + for (auto &ur : uranges) { + cnt += ur.end - ur.start + 1; + cnt += buildPartialHdr(ur.start, + ur.end, + filesize, + (char *) "123456").size(); - } else if (xrdresp != kXR_ok) { + } + cnt += buildPartialHdrEnd((char *) "123456").size(); + std::string header = "Content-Type: multipart/byteranges; boundary=123456"; + if (!m_digest_header.empty()) { + header += "\n"; + header += m_digest_header; + } + + if (m_transfer_encoding_chunked && m_trailer_headers) { + prot->StartChunkedResp(206, NULL, header.c_str(), -1, keepalive); + } else { + prot->SendSimpleResp(206, NULL, header.c_str(), NULL, cnt, keepalive); + } + return 0; + + + } else { // xrdresp indicates an error occurred - // If it's a dir then we are in the wrong place and we did the wrong thing. - //if (xrderrcode == 3016) { - // fileflags &= kXR_isDir; - // reqstate--; - // return 0; - //} prot->SendSimpleResp(httpStatusCode, NULL, NULL, httpStatusText.c_str(), httpStatusText.length(), false); return -1; } - // Remaining case: reqstate == 2 and we didn't ask for a digest (should be a read). + // Case should not be reachable + return -1; } - // fallthrough default: //read or readv { - // If we are postprocessing a close, potentially send out informational trailers - if ((ntohs(xrdreq.header.requestid) == kXR_close) || - ((reqstate == 3) && (ntohs(xrdreq.header.requestid) == kXR_readv))) + if ((ntohs(xrdreq.header.requestid) == kXR_close) || readClosing) { - + const XrdHttpReadRangeHandler::Error &rrerror = readRangeHandler.getError(); + if (rrerror) { + httpStatusCode = rrerror.httpRetCode; + httpStatusText = rrerror.errMsg; + } + if (m_transfer_encoding_chunked && m_trailer_headers) { if (prot->ChunkRespHeader(0)) return -1; @@ -2280,7 +2264,8 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { return -1; } - return keepalive ? 1 : -1; + if (rrerror) return -1; + return keepalive ? 1 : -1; } // On error, we can only send out a message if trailers are enabled and the @@ -2313,91 +2298,30 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { } } - // Prevent scenario where data is expected but none is actually read - // E.g. Accessing files which return the results of a script - if ((ntohs(xrdreq.header.requestid) == kXR_read) && - (reqstate > 2) && (iovN == 0)) { - TRACEI(REQ, "Stopping request because more data is expected " - "but no data has been read."); - return -1; - } - TRACEI(REQ, "Got data vectors to send:" << iovN); - if (ntohs(xrdreq.header.requestid) == kXR_readv) { - // Readv case, we must take out each individual header and format it according to the http rules - readahead_list *l; - char *p; - int len; - - // Cycle on all the data that is coming from the server - for (int i = 0; i < iovN; i++) { - - for (p = (char *) iovP[i].iov_base; p < (char *) iovP[i].iov_base + iovP[i].iov_len;) { - l = (readahead_list *) p; - len = ntohl(l->rlen); - - // Now we have a chunk coming from the server. This may be a partial chunk - - if (rwOpPartialDone == 0) { - string s = buildPartialHdr(rwOps[rwOpDone].bytestart, - rwOps[rwOpDone].byteend, - filesize, - (char *) "123456"); - - TRACEI(REQ, "Sending multipart: " << rwOps[rwOpDone].bytestart << "-" << rwOps[rwOpDone].byteend); - if (prot->SendData((char *) s.c_str(), s.size())) return -1; - } - // Send all the data we have - if (prot->SendData(p + sizeof (readahead_list), len)) return -1; - - // If we sent all the data relative to the current original chunk request - // then pass to the next chunk, otherwise wait for more data - rwOpPartialDone += len; - if (rwOpPartialDone >= rwOps[rwOpDone].byteend - rwOps[rwOpDone].bytestart + 1) { - rwOpDone++; - rwOpPartialDone = 0; - } - - p += sizeof (readahead_list); - p += len; - - } - } - - if (rwOpDone == rwOps.size()) { - string s = buildPartialHdrEnd((char *) "123456"); - if (prot->SendData((char *) s.c_str(), s.size())) return -1; - } + XrdHttpIOList received; + getReadResponse(received); + int rc; + if (readRangeHandler.isSingleRange()) { + rc = sendReadResponseSingleRange(received); } else { - // Send chunked encoding header - if (m_transfer_encoding_chunked && m_trailer_headers) { - int sum = 0; - for (int i = 0; i < iovN; i++) sum += iovP[i].iov_len; - prot->ChunkRespHeader(sum); - } - for (int i = 0; i < iovN; i++) { - if (prot->SendData((char *) iovP[i].iov_base, iovP[i].iov_len)) return -1; - writtenbytes += iovP[i].iov_len; - } - if (m_transfer_encoding_chunked && m_trailer_headers) { - prot->ChunkRespFooter(); - } + rc = sendReadResponsesMultiRanges(received); } - - // Let's make sure that we avoid sending the same data twice, - // in the case where PostProcessHTTPReq is invoked again - this->iovN = 0; - + if (rc) { + // make sure readRangeHandler will trigger close + // of file after next NextReadList(). + readRangeHandler.NotifyError(); + } + return 0; - } + } // end read or readv } // switch reqstate - - } + } // End handling of the open-read-close case break; @@ -2419,7 +2343,7 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { fopened = true; // We try to completely fill up our buffer before flushing - prot->ResumeBytes = min(length - writtenbytes, (long long) prot->BuffAvailable()); + prot->ResumeBytes = std::min(length - writtenbytes, (long long) prot->BuffAvailable()); if (sendcontinue) { prot->SendSimpleResp(100, NULL, NULL, 0, 0, keepalive); @@ -2446,7 +2370,7 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { } // We try to completely fill up our buffer before flushing - prot->ResumeBytes = min(length - writtenbytes, (long long) prot->BuffAvailable()); + prot->ResumeBytes = std::min(length - writtenbytes, (long long) prot->BuffAvailable()); return 0; } @@ -2557,7 +2481,7 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { /* The entry is filled. */ - string p; + std::string p; stringresp += "\n"; char *estr = escapeXML(e.path.c_str()); @@ -2611,7 +2535,7 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { // If this was the last bunch of entries, send the buffer and empty it immediately if ((depth == 0) || !(e.flags & kXR_isDir)) { - string s = "\n\n"; + std::string s = "\n\n"; stringresp.insert(0, s); stringresp += "\n"; prot->SendSimpleResp(207, (char *) "Multi-Status", (char *) "Content-Type: text/xml; charset=\"utf-8\"", @@ -2673,7 +2597,7 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { */ - string p = resource.c_str(); + std::string p = resource.c_str(); if (*p.rbegin() != '/') p += "/"; p += e.path; @@ -2735,7 +2659,7 @@ int XrdHttpReq::PostProcessHTTPReq(bool final_) { // If this was the last bunch of entries, send the buffer and empty it immediately if (final_) { - string s = "\n\n"; + std::string s = "\n\n"; stringresp.insert(0, s); stringresp += "\n"; prot->SendSimpleResp(207, (char *) "Multi-Status", (char *) "Content-Type: text/xml; charset=\"utf-8\"", @@ -2810,10 +2734,8 @@ void XrdHttpReq::reset() { TRACE(REQ, " XrdHttpReq request ended."); //if (xmlbody) xmlFreeDoc(xmlbody); - rwOps.clear(); - rwOps_split.clear(); - rwOpDone = 0; - rwOpPartialDone = 0; + readRangeHandler.reset(); + readClosing = false; writtenbytes = 0; etext.clear(); redirdest = ""; @@ -2829,8 +2751,8 @@ void XrdHttpReq::reset() { depth = 0; xrdresp = kXR_noResponsesYet; xrderrcode = kXR_noErrorYet; - if (ralist) free(ralist); - ralist = 0; + ralist.clear(); + ralist.shrink_to_fit(); request = rtUnset; resource = ""; @@ -2842,6 +2764,7 @@ void XrdHttpReq::reset() { m_req_cksum = nullptr; m_resource_with_digest = ""; + m_user_agent = ""; headerok = false; keepalive = true; @@ -2894,3 +2817,170 @@ void XrdHttpReq::getfhandle() { (int) fhandle[0] << ":" << (int) fhandle[1] << ":" << (int) fhandle[2] << ":" << (int) fhandle[3]); } + +void XrdHttpReq::getReadResponse(XrdHttpIOList &received) { + received.clear(); + + if (ntohs(xrdreq.header.requestid) == kXR_readv) { + readahead_list *l; + char *p; + kXR_int32 len; + + // Cycle on all the data that is coming from the server + for (int i = 0; i < iovN; i++) { + + for (p = (char *) iovP[i].iov_base; p < (char *) iovP[i].iov_base + iovP[i].iov_len;) { + l = (readahead_list *) p; + len = ntohl(l->rlen); + + received.emplace_back(p+sizeof(readahead_list), -1, len); + + p += sizeof (readahead_list); + p += len; + + } + } + return; + } + + // kXR_read result + for (int i = 0; i < iovN; i++) { + received.emplace_back((char*)iovP[i].iov_base, -1, iovP[i].iov_len); + } + +} + +int XrdHttpReq::sendReadResponsesMultiRanges(const XrdHttpIOList &received) { + + if (received.size() == 0) { + bool start, finish; + if (readRangeHandler.NotifyReadResult(0, nullptr, start, finish) < 0) { + return -1; + } + return 0; + } + + // user is expecting multiple ranges, we must be prepared to send an + // individual header for each and format it according to the http rules + + struct rinfo { + bool start; + bool finish; + const XrdOucIOVec2 *ci; + const XrdHttpReadRangeHandler::UserRange *ur; + std::string st_header; + std::string fin_header; + }; + + // report each received byte chunk to the range handler and record the details + // of original user range it related to and if starts a range or finishes all. + // also sum the total of the headers and data which need to be sent to the user, + // in case we need it for chunked transfer encoding + std::vector rvec; + off_t sum_len = 0; + + rvec.reserve(received.size()); + + for(const auto &rcv: received) { + rinfo rentry; + bool start, finish; + const XrdHttpReadRangeHandler::UserRange *ur; + + if (readRangeHandler.NotifyReadResult(rcv.size, &ur, start, finish) < 0) { + return -1; + } + rentry.ur = ur; + rentry.start = start; + rentry.finish = finish; + rentry.ci = &rcv; + + if (start) { + std::string s = buildPartialHdr(ur->start, + ur->end, + filesize, + (char *) "123456"); + + rentry.st_header = s; + sum_len += s.size(); + } + + sum_len += rcv.size; + + if (finish) { + std::string s = buildPartialHdrEnd((char *) "123456"); + rentry.fin_header = s; + sum_len += s.size(); + } + + rvec.push_back(rentry); + } + + + // Send chunked encoding header + if (m_transfer_encoding_chunked && m_trailer_headers) { + prot->ChunkRespHeader(sum_len); + } + + // send the user the headers / data + for(const auto &rentry: rvec) { + + if (rentry.start) { + TRACEI(REQ, "Sending multipart: " << rentry.ur->start << "-" << rentry.ur->end); + if (prot->SendData((char *) rentry.st_header.c_str(), rentry.st_header.size())) { + return -1; + } + } + + // Send all the data we have + if (prot->SendData((char *) rentry.ci->data, rentry.ci->size)) { + return -1; + } + + if (rentry.finish) { + if (prot->SendData((char *) rentry.fin_header.c_str(), rentry.fin_header.size())) { + return -1; + } + } + } + + // Send chunked encoding footer + if (m_transfer_encoding_chunked && m_trailer_headers) { + prot->ChunkRespFooter(); + } + + return 0; +} + +int XrdHttpReq::sendReadResponseSingleRange(const XrdHttpIOList &received) { + // single range http transfer + + if (received.size() == 0) { + bool start, finish; + if (readRangeHandler.NotifyReadResult(0, nullptr, start, finish) < 0) { + return -1; + } + return 0; + } + + off_t sum = 0; + // notify the range handler and return if error + for(const auto &rcv: received) { + bool start, finish; + if (readRangeHandler.NotifyReadResult(rcv.size, nullptr, start, finish) < 0) { + return -1; + } + sum += rcv.size; + } + + // Send chunked encoding header + if (m_transfer_encoding_chunked && m_trailer_headers) { + prot->ChunkRespHeader(sum); + } + for(const auto &rcv: received) { + if (prot->SendData((char *) rcv.data, rcv.size)) return -1; + } + if (m_transfer_encoding_chunked && m_trailer_headers) { + prot->ChunkRespFooter(); + } + return 0; +} diff --git a/src/XrdHttp/XrdHttpReq.hh b/src/XrdHttp/XrdHttpReq.hh index 81e50683eea..ed56ede2066 100644 --- a/src/XrdHttp/XrdHttpReq.hh +++ b/src/XrdHttp/XrdHttpReq.hh @@ -44,6 +44,7 @@ #include "XProtocol/XProtocol.hh" #include "XrdXrootd/XrdXrootdBridge.hh" #include "XrdHttpChecksumHandler.hh" +#include "XrdHttpReadRangeHandler.hh" #include #include @@ -54,14 +55,6 @@ -#define READV_MAXCHUNKS 512 -#define READV_MAXCHUNKSIZE (1024*128) - -struct ReadWriteOp { - // < 0 means "not specified" - long long bytestart; - long long byteend; -}; struct DirListInfo { std::string path; @@ -81,6 +74,9 @@ private: int httpStatusCode; std::string httpStatusText; + // The value of the user agent, if specified + std::string m_user_agent; + // Whether transfer encoding was requested. bool m_transfer_encoding_chunked; long long m_current_chunk_offset; @@ -94,9 +90,9 @@ private: // after a response body has started bool m_status_trailer{false}; - int parseContentRange(char *); int parseHost(char *); - int parseRWOp(char *); + + void parseScitag(const std::string & val); //xmlDocPtr xmlbody; /* the resulting document tree */ XrdHttpProtocol *prot; @@ -126,6 +122,19 @@ private: // Sanitize the resource from http[s]://[host]/ questionable prefix void sanitizeResourcePfx(); + // parses the iovN data pointers elements as either a kXR_read or kXR_readv + // response and fills out a XrdHttpIOList with the corresponding length and + // buffer pointers. File offsets from kXR_readv responses are not recorded. + void getReadResponse(XrdHttpIOList &received); + + // notifies the range handler of receipt of bytes and sends the client + // the data. + int sendReadResponseSingleRange(const XrdHttpIOList &received); + + // notifies the range handler of receipt of bytes and sends the client + // the data and necessary headers, assuming multipart/byteranges content type. + int sendReadResponsesMultiRanges(const XrdHttpIOList &received); + /** * Extract a comma separated list of checksums+metadata into a vector * @param checksumList the list like "0:sha1, 1:adler32, 2:md5" @@ -142,17 +151,18 @@ private: static void determineXRootDChecksumFromUserDigest(const std::string & userDigest, std::vector & xrootdChecksums); public: - XrdHttpReq(XrdHttpProtocol *protinstance) : keepalive(true) { + XrdHttpReq(XrdHttpProtocol *protinstance, const XrdHttpReadRangeHandler::Configuration &rcfg) : + readRangeHandler(rcfg), keepalive(true) { prot = protinstance; length = 0; //xmlbody = 0; depth = 0; - ralist = 0; opaque = 0; writtenbytes = 0; fopened = false; headerok = false; + mScitag = -1; }; virtual ~XrdHttpReq(); @@ -169,8 +179,8 @@ public: int parseBody(char *body, long long len); /// Prepare the buffers for sending a readv request - int ReqReadV(); - readahead_list *ralist; + int ReqReadV(const XrdHttpIOList &cl); + std::vector ralist; /// Build a partial header for a multipart response std::string buildPartialHdr(long long bytestart, long long byteend, long long filesize, char *token); @@ -182,6 +192,11 @@ public: // NOTE: this function assumes that the strings are unquoted, and will quote them void appendOpaque(XrdOucString &s, XrdSecEntity *secent, char *hash, time_t tnow); + void addCgi(const std::string & key, const std::string & value); + + // Return the current user agent; if none has been specified, returns an empty string + const std::string &userAgent() const {return m_user_agent;} + // ---------------- // Description of the request. The header/body parsing // is supposed to populate these fields, for fast access while @@ -224,13 +239,9 @@ public: /// Tells if we have finished reading the header bool headerok; - - // This can be largely optimized... - /// The original list of multiple reads to perform - std::vector rwOps; - /// The new list got from chunking the original req respecting the xrootd - /// max sizes etc. - std::vector rwOps_split; + /// Tracking the next ranges of data to read during GET + XrdHttpReadRangeHandler readRangeHandler; + bool readClosing; bool keepalive; long long length; // Total size from client for PUT; total length of response TO client for GET. @@ -299,6 +310,8 @@ public: /// In a long write, we track where we have arrived long long writtenbytes; + int mScitag; + diff --git a/src/XrdHttp/XrdHttpUtils.cc b/src/XrdHttp/XrdHttpUtils.cc index 78fa4f26ab8..34932988827 100644 --- a/src/XrdHttp/XrdHttpUtils.cc +++ b/src/XrdHttp/XrdHttpUtils.cc @@ -93,14 +93,14 @@ int parseURL(char *url, char *host, int &port, char **path) { *path = p2; char buf[256]; - int l = min((int)(p2 - p), (int)sizeof (buf)); + int l = std::min((int)(p2 - p), (int)sizeof (buf)); strncpy(buf, p, l); buf[l] = '\0'; // Now look for : p = strchr(buf, ':'); if (p) { - int l = min((int)(p - buf), (int)sizeof (buf)); + int l = std::min((int)(p - buf), (int)sizeof (buf)); strncpy(host, buf, l); host[l] = '\0'; diff --git a/src/XrdHttp/XrdHttpUtils.hh b/src/XrdHttp/XrdHttpUtils.hh index 67d334bf11d..3b5ff6c2fe9 100644 --- a/src/XrdHttp/XrdHttpUtils.hh +++ b/src/XrdHttp/XrdHttpUtils.hh @@ -37,6 +37,9 @@ #include "XProtocol/XPtypes.hh" #include "XrdSec/XrdSecEntity.hh" +#include "XrdOuc/XrdOucIOVec.hh" +#include +#include #ifndef XRDHTTPUTILS_HH #define XRDHTTPUTILS_HH @@ -89,6 +92,8 @@ char *unquote(char *str); // Escape a string and return a new one char *escapeXML(const char *str); +typedef std::vector XrdHttpIOList; + #endif /* XRDHTTPUTILS_HH */ diff --git a/src/XrdIsal.cmake b/src/XrdIsal.cmake index f110df69dc3..496fd29dbc7 100644 --- a/src/XrdIsal.cmake +++ b/src/XrdIsal.cmake @@ -29,18 +29,17 @@ ExternalProject_add(isa-l BUILD_COMMAND make -j ${CMAKE_BUILD_PARALLEL_LEVEL} INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory ${ISAL_ROOT}/include ${ISAL_ROOT}/isa-l BUILD_BYPRODUCTS ${ISAL_LIBRARY} ${ISAL_INCLUDE_DIRS} + LOG_DOWNLOAD 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 ) -add_library(isal STATIC IMPORTED) - -set(ISAL_LIBRARIES isal) +add_library(isal INTERFACE) add_dependencies(isal isa-l) -set_target_properties(isal - PROPERTIES - IMPORTED_LOCATION "${ISAL_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "$" -) +target_link_libraries(isal INTERFACE $) +target_include_directories(isal INTERFACE $) + +set(ISAL_LIBRARIES isal CACHE INTERNAL "" FORCE) +set(ISAL_INCLUDE_DIRS ${ISAL_INCLUDE_DIRS} CACHE INTERNAL "" FORCE) # Emulate what happens when find_package(isal) succeeds find_package_handle_standard_args(isal diff --git a/src/XrdMacaroons.cmake b/src/XrdMacaroons.cmake index 837bfe22e16..edf50395e2c 100644 --- a/src/XrdMacaroons.cmake +++ b/src/XrdMacaroons.cmake @@ -1,4 +1,3 @@ -include( XRootDCommon ) #------------------------------------------------------------------------------- # Modules @@ -10,8 +9,6 @@ set( LIB_XRD_MACAROONS XrdMacaroons-${PLUGIN_VERSION} ) #------------------------------------------------------------------------------- if( BUILD_MACAROONS ) - include_directories(${MACAROONS_INCLUDES} ${JSON_INCLUDE_DIRS}) - add_library( ${LIB_XRD_MACAROONS} MODULE @@ -21,7 +18,8 @@ if( BUILD_MACAROONS ) XrdMacaroons/XrdMacaroonsConfigure.cc) target_link_libraries( - ${LIB_XRD_MACAROONS} ${CMAKE_DL_LIBS} + ${LIB_XRD_MACAROONS} + PRIVATE XrdHttpUtils XrdUtils XrdServer @@ -29,7 +27,11 @@ if( BUILD_MACAROONS ) ${MACAROONS_LIB} ${JSON_LIBRARIES} ${XROOTD_HTTP_LIB} - OpenSSL::Crypto) + OpenSSL::Crypto + ${CMAKE_DL_LIBS}) + + target_include_directories(${LIB_XRD_MACAROONS} + PRIVATE ${MACAROONS_INCLUDES} ${JSON_INCLUDE_DIRS}) if( MacOSX ) SET( MACAROONS_LINK_FLAGS "-Wl") @@ -40,8 +42,6 @@ if( BUILD_MACAROONS ) set_target_properties( ${LIB_XRD_MACAROONS} PROPERTIES - INTERFACE_LINK_LIBRARIES "" - LINK_INTERFACE_LIBRARIES "" LINK_FLAGS "${MACAROONS_LINK_FLAGS}") #----------------------------------------------------------------------------- diff --git a/src/XrdMacaroons/XrdMacaroonsAuthz.hh b/src/XrdMacaroons/XrdMacaroonsAuthz.hh index acf88bc9c73..5ab52df1576 100644 --- a/src/XrdMacaroons/XrdMacaroonsAuthz.hh +++ b/src/XrdMacaroons/XrdMacaroonsAuthz.hh @@ -43,7 +43,7 @@ public: // Macaroons don't have a concept off an "issuers"; return an empty // list. - virtual Issuers IssuerList() {return Issuers();} + virtual Issuers IssuerList() override {return Issuers();} private: XrdAccPrivs OnMissing(const XrdSecEntity *Entity, diff --git a/src/XrdNet/XrdNetAddrInfo.cc b/src/XrdNet/XrdNetAddrInfo.cc index cfbe3b54a9d..12424dd29dd 100644 --- a/src/XrdNet/XrdNetAddrInfo.cc +++ b/src/XrdNet/XrdNetAddrInfo.cc @@ -53,6 +53,16 @@ #endif #endif +// The following tests for Unique Local Addresses (ULA) which Linux does not +// provide. The SITELOCAL macro only tests for the now deprecated non-routable +// addresses (RFC 3879). So, we need to implement the ULA test ourselves. +// Technically, only addresses starting with prefix 0xfd are ULA useable but +// RFC 4193 doesn't explicitly prohibit ULA's that start with 0xfc which may +// be used for registered ULA's in the future. So we test for both. +// +#define IN6_IS_ADDR_UNIQLOCAL(a) \ + ( ((const uint8_t *)(a))[0] == 0xfc || ((const uint8_t *)(a))[0] == 0xfd ) + /******************************************************************************/ /* S t a t i c M e m b e r s */ /******************************************************************************/ @@ -198,6 +208,7 @@ bool XrdNetAddrInfo::isPrivate() ipV4 = (unsigned char *)&IP.v6.sin6_addr.s6_addr32[3]; else {if ((IN6_IS_ADDR_LINKLOCAL(&IP.v6.sin6_addr)) || (IN6_IS_ADDR_SITELOCAL(&IP.v6.sin6_addr)) + || (IN6_IS_ADDR_UNIQLOCAL(&IP.v6.sin6_addr)) || (IN6_IS_ADDR_LOOPBACK (&IP.v6.sin6_addr))) return true; return false; } diff --git a/src/XrdNet/XrdNetIF.cc b/src/XrdNet/XrdNetIF.cc index 8d004c32198..e7c0a51fbe5 100644 --- a/src/XrdNet/XrdNetIF.cc +++ b/src/XrdNet/XrdNetIF.cc @@ -409,7 +409,7 @@ int XrdNetIF::GetDest(char *dest, int dlen, ifType ifT, bool prefn) /* G e t I F */ /******************************************************************************/ -#define prtaddr(x) cerr <<"Addr!!! " << *x <Say("Config invalid: pmark 'use firefly' requires " - "specifying 'ffdest'!"); - fatal = true; + {if (ffPortD || ffPortO) + {useFFly = true; + if (!ffPortO) ffPortO = ffPORT; + } else { + useFFly = false; + eLog->Say("Config warning: firefly disabled; " + "configuration incomplete!"); return 0; } - } + } else if (useFFly && !ffPortO) ffPortO = ffPORT; // Resolve trace and debug settings // @@ -901,7 +897,7 @@ bool XrdNetPMarkCfg::LoadJson(char *buff) for (auto it : j_exp) {std::string expName = it["expName"].get(); if (expName.empty()) continue; - if (!it["expId"].is_number() || it["expId"] < 0 || it["expId"] > maxExpID) + if (!it["expId"].is_number() || it["expId"] < minExpID || it["expId"] > maxExpID) {eDest->Say("Config warning: ignoring experiment '", expName.c_str(), "'; associated ID is invalid."); continue; @@ -925,7 +921,7 @@ bool XrdNetPMarkCfg::LoadJson(char *buff) {std::string actName = j_acts[i]["activityName"].get(); if (actName.empty()) continue; if (!j_acts[i]["activityId"].is_number() - || j_acts[i]["activityId"] < 0 + || j_acts[i]["activityId"] < minActID || j_acts[i]["activityId"] > maxActID) {eDest->Say("Config warning:", "ignoring ", expName.c_str(), " actitivity '", actName.c_str(), @@ -1042,6 +1038,9 @@ do{if (!strcmp("debug", val) || !strcmp("nodebug", val)) continue; } + // We accept 'origin' as a dest for backward compatibility. That is the + // enforced default should 'use firefly' be specified. + // if (!strcmp("ffdest", val)) {const char *addtxt = ""; char *colon, *comma; diff --git a/src/XrdOfs/XrdOfs.cc b/src/XrdOfs/XrdOfs.cc index 03c644442b1..6d829977344 100644 --- a/src/XrdOfs/XrdOfs.cc +++ b/src/XrdOfs/XrdOfs.cc @@ -173,6 +173,7 @@ XrdOfs::XrdOfs() : dMask{0000,0775}, fMask{0000,0775}, // Legacy // Establish defaults // ofsConfig = 0; + FSctl_PC = 0; FSctl_PI = 0; Authorization = 0; Finder = 0; diff --git a/src/XrdOfs/XrdOfs.hh b/src/XrdOfs/XrdOfs.hh index 5e9fecde0e0..eeb065ed374 100644 --- a/src/XrdOfs/XrdOfs.hh +++ b/src/XrdOfs/XrdOfs.hh @@ -446,6 +446,7 @@ const char *Split(const char *Args, const char **Opq, char *Path, int Plen); private: char *myRole; +XrdOfsFSctl_PI *FSctl_PC; // ->FSctl plugin (cache specific) XrdOfsFSctl_PI *FSctl_PI; // ->FSctl plugin XrdAccAuthorize *Authorization; // ->Authorization Service XrdCmsClient *Balancer; // ->Cluster Local Interface diff --git a/src/XrdOfs/XrdOfsConfig.cc b/src/XrdOfs/XrdOfsConfig.cc index 79b5bb7b149..0b9873a6d64 100644 --- a/src/XrdOfs/XrdOfsConfig.cc +++ b/src/XrdOfs/XrdOfsConfig.cc @@ -53,6 +53,7 @@ #include "XrdOfs/XrdOfsConfigCP.hh" #include "XrdOfs/XrdOfsConfigPI.hh" #include "XrdOfs/XrdOfsEvs.hh" +#include "XrdOfs/XrdOfsFSctl_PI.hh" #include "XrdOfs/XrdOfsPoscq.hh" #include "XrdOfs/XrdOfsStats.hh" #include "XrdOfs/XrdOfsTPC.hh" @@ -85,6 +86,8 @@ extern XrdOfsStats OfsStats; extern XrdSysTrace OfsTrace; + +extern XrdOfs* XrdOfsFS; class XrdOss; extern XrdOss *XrdOfsOss; @@ -292,6 +295,12 @@ int XrdOfs::Configure(XrdSysError &Eroute, XrdOucEnv *EnvInfo) { } } +// If a cache has been configured then that cache may want to interact with +// the cache-specific FSctl() operation. We check if a plugin was provided. +// + if (ossFeatures & XRDOSS_HASCACH) + FSctl_PC = (XrdOfsFSctl_PI*)EnvInfo->GetPtr("XrdFSCtl_PC*"); + // Configure third party copy phase 2, but only if we are not a manager. // if ((Options & ThirdPC) && !(Options & isManager)) NoGo |= ConfigTPC(Eroute); @@ -318,6 +327,19 @@ int XrdOfs::Configure(XrdSysError &Eroute, XrdOucEnv *EnvInfo) { NoGo = 1; } +// Initialize the cache FSctl handler if we have one. The same deferal applies. +// + if (FSctl_PC) + {struct XrdOfsFSctl_PI::Plugins thePI = {Authorization, Finder, + XrdOfsOss, XrdOfsFS}; + XrdOucEnv pcEnv; + pcEnv.PutPtr("XrdOfsHandle*", dummyHandle); + if (!FSctl_PC->Configure(ConfigFN, 0, &pcEnv, thePI)) + {Eroute.Emsg("Config", "Unable to configure cache FSctl handler."); + NoGo = 1; + } + } + // Initialize th Evr object if we are an actual server // if (!(Options & isManager) && !evrObject.Init(Balancer)) NoGo = 1; @@ -738,7 +760,7 @@ char *XrdOfs::ConfigTPCDir(XrdSysError &Eroute, const char *sfx, return 0; } -// list the contents of teh directory +// list the contents of the directory // XrdOucNSWalk nsWalk(&Eroute, aPath, 0, nswOpt); XrdOucNSWalk::NSEnt *nsX, *nsP = nsWalk.Index(rc); diff --git a/src/XrdOfs/XrdOfsConfigPI.cc b/src/XrdOfs/XrdOfsConfigPI.cc index 2682bccfcb9..5e4c25edc62 100644 --- a/src/XrdOfs/XrdOfsConfigPI.cc +++ b/src/XrdOfs/XrdOfsConfigPI.cc @@ -828,7 +828,8 @@ bool XrdOfsConfigPI::SetupAuth(XrdOucEnv *envP) (XrdSysLogger *lp, const char *cfn, const char *parm, XrdVersionInfo &vInfo); - XrdAccAuthorizeObject_t ep; + XrdAccAuthorizeObject_t ep1; + XrdAccAuthorizeObject2_t ep2; char *AuthLib = LP[PIX(theAutLib)].lib; char *AuthParms = LP[PIX(theAutLib)].parms; @@ -841,18 +842,24 @@ bool XrdOfsConfigPI::SetupAuth(XrdOucEnv *envP) return AddLibAut(envP); } -// Create a plugin object +// Create a plugin object. It will be version 2 or version 1, in that order // {XrdOucPinLoader myLib(Eroute, urVer, "authlib", AuthLib); - ep = (XrdAccAuthorizeObject_t)(myLib.Resolve("XrdAccAuthorizeObject")); - if (!ep) return false; + ep2 = (XrdAccAuthorizeObject2_t)(myLib.Resolve("XrdAccAuthorizeObject2")); + if (!ep2) + {ep1 = (XrdAccAuthorizeObject_t)(myLib.Resolve("XrdAccAuthorizeObject")); + if (!ep1) return false; + if (!(autPI = ep1(Eroute->logger(), ConfigFN, AuthParms))) return false; + } else { + if (!(autPI = ep2(Eroute->logger(), ConfigFN, AuthParms, envP))) + return false; + } if (strcmp(AuthLib, myLib.Path())) {free(AuthLib); AuthLib = LP[PIX(theAutLib)].lib = strdup(myLib.Path());} } -// Get the Object now +// Process additional wrapper objects now // - if (!(autPI = ep(Eroute->logger(), ConfigFN, AuthParms))) return false; return AddLibAut(envP); } diff --git a/src/XrdOfs/XrdOfsFSctl.cc b/src/XrdOfs/XrdOfsFSctl.cc index 6a1bf8e6dab..527eb9eeb80 100644 --- a/src/XrdOfs/XrdOfsFSctl.cc +++ b/src/XrdOfs/XrdOfsFSctl.cc @@ -265,9 +265,21 @@ int XrdOfs::FSctl(const int cmd, XrdOucErrInfo &eInfo, const XrdSecEntity *client) { -// If we have a plugin to handle this, use it. + EPNAME("FSctl"); + +// If this is the cache-specfic we need to do a lot more work. Otherwise this +// is a simple case of wheter we have a plug-in for this or not. // - if (FSctl_PI) return FSctl_PI->FSctl(cmd, args, eInfo, client); + if (cmd == SFS_FSCTL_PLUGXC) + {if (FSctl_PC) + {if (args.Arg2Len == -2) + {XrdOucEnv pc_Env(args.ArgP[1] ? args.ArgP[1] : 0, 0, client); + AUTHORIZE(client,&pc_Env,AOP_Read,"FSctl",args.ArgP[0],eInfo); + } + return FSctl_PC->FSctl(cmd, args, eInfo, client); + } + } + else if (FSctl_PI) return FSctl_PI->FSctl(cmd, args, eInfo, client); // Operation is not supported // diff --git a/src/XrdOfs/XrdOfsPrepGPI.cc b/src/XrdOfs/XrdOfsPrepGPI.cc index 2b9b9a1d251..c40f7feede2 100644 --- a/src/XrdOfs/XrdOfsPrepGPI.cc +++ b/src/XrdOfs/XrdOfsPrepGPI.cc @@ -13,6 +13,7 @@ #include "XrdOss/XrdOss.hh" #include "XrdOuc/XrdOuca2x.hh" +#include "XrdOuc/XrdOucBuffer.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucErrInfo.hh" #include "XrdOuc/XrdOucGatherConf.hh" @@ -51,6 +52,8 @@ namespace XrdOfsPrepGPIReal { XrdSysMutex gpiMutex; +XrdOucBuffPool *bPool = 0;; + XrdOss *ossP = 0; XrdScheduler *schedP = 0; XrdSysError *eLog = 0; @@ -62,6 +65,7 @@ int qryWait = 0; // Ditto static const int qryMaxWT = 33; // Maximum wait time int maxFiles = 48; +int maxResp = XrdOucEI::Max_Error_Len; bool addCGI = false; bool Debug = false; @@ -619,6 +623,10 @@ int PrepGPI::query( XrdSfsPrep &pargs, const XrdSecEntity *client) { EPNAME("Query"); + struct OucBuffer {XrdOucBuffer *pBuff; + OucBuffer() : pBuff(0) {} + ~OucBuffer() {if (pBuff) pBuff->Recycle();} + } OucBuff; const char *tid = (client ? client->tident : "anon"); int rc, bL; char *bP = eInfo.getMsgBuff(bL); @@ -629,14 +637,24 @@ int PrepGPI::query( XrdSfsPrep &pargs, if (!(okReq & okQuery)) {PrepRequest *rPP, *rP; if (reqFind(pargs.reqid, rPP, rP)) - {bL = snprintf(bP, bL, "Request %s queued.", pargs.reqid); + {bL = snprintf(bP, bL, "Request %s queued.", pargs.reqid)+1; } else { - bL = snprintf(bP, bL, "Request %s not queued.", pargs.reqid); + bL = snprintf(bP, bL, "Request %s not queued.", pargs.reqid)+1; } eInfo.setErrCode(bL); return SFS_DATA; } +// Allocate a buffer if need be +// + if (bPool) + {OucBuff.pBuff = bPool->Alloc(maxResp); + if (OucBuff.pBuff) + {bP = OucBuff.pBuff->Buffer(); + bL = maxResp; + } + } + // Get a request request object // PrepRequest *rP = Assemble(rc, tid, "query", pargs, ""); @@ -679,7 +697,11 @@ int PrepGPI::query( XrdSfsPrep &pargs, // Return response // - eInfo.setErrCode(rc); + if (!OucBuff.pBuff) eInfo.setErrCode(rc); + else {OucBuff.pBuff->SetLen(rc); + eInfo.setErrInfo(rc, OucBuff.pBuff); + OucBuff.pBuff = 0; + } return SFS_DATA; } } @@ -789,7 +811,7 @@ int PrepGPI::Xeq(PrepRequest *rP) /******************************************************************************/ // Parameters: -admit [-cgi] [-maxfiles [-maxreq ] -// [-maxquery ] [-pfn] -run +// [-maxquery ] [-maxresp ] [-pfn] -run // // : cancel | evict | prep | query | stage // : [,] @@ -878,6 +900,16 @@ XrdOfsPrepare *XrdOfsgetPrepare(XrdOfsgetPrepareArguments) if (XrdOuca2x::a2i(*eLog, "PrepPGI -maxreq", tokP, &maxReq, 1, 64)) return 0; } + else if (Token == "-maxresp") + {if (!(tokP = gpiConf.GetToken()) || *tokP == '-') + {eLog->Emsg("PrepGPI", "-maxresp argument not specified."); + return 0; + } + long long rspsz; + if (XrdOuca2x::a2sz(*eLog, "PrepPGI -maxresp", tokP, + &rspsz, 2048, 16777216)) return 0; + maxResp = static_cast(rspsz); + } else if (Token == "-pfn") usePFN = true; else if (Token == "-run") {if (!(tokP = gpiConf.GetToken()) || *tokP == '-') @@ -905,6 +937,11 @@ XrdOfsPrepare *XrdOfsgetPrepare(XrdOfsgetPrepareArguments) return 0; } +// Create a buffer pool for query responses if we need to +// + if (maxResp > (int)XrdOucEI::Max_Error_Len) + bPool = new XrdOucBuffPool(maxResp, maxResp); + // Set final debug flags // if (!Debug) Debug = getenv("XRDDEBUG") != 0; diff --git a/src/XrdOfs/XrdOfsTPC.cc b/src/XrdOfs/XrdOfsTPC.cc index 6f19b09bdba..9f2b576ad5a 100644 --- a/src/XrdOfs/XrdOfsTPC.cc +++ b/src/XrdOfs/XrdOfsTPC.cc @@ -154,7 +154,7 @@ char *XrdOfsTPC::cPath = 0; namespace { -atomic_int rpInst = {0}; +std::atomic rpInst = {0}; } /******************************************************************************/ diff --git a/src/XrdOssCsi.cmake b/src/XrdOssCsi.cmake index 3ef4526f95d..30adcf1f013 100644 --- a/src/XrdOssCsi.cmake +++ b/src/XrdOssCsi.cmake @@ -29,16 +29,11 @@ add_library( target_link_libraries( ${LIB_XRD_OSSCSI} + PRIVATE XrdUtils XrdServer ${CMAKE_THREAD_LIBS_INIT} ) -set_target_properties( - ${LIB_XRD_OSSCSI} - PROPERTIES - INTERFACE_LINK_LIBRARIES "" - LINK_INTERFACE_LIBRARIES "" ) - #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- diff --git a/src/XrdOssCsi/XrdOssCsiTagstoreFile.hh b/src/XrdOssCsi/XrdOssCsiTagstoreFile.hh index 0202579ccb4..2ae948048ac 100644 --- a/src/XrdOssCsi/XrdOssCsiTagstoreFile.hh +++ b/src/XrdOssCsi/XrdOssCsiTagstoreFile.hh @@ -34,23 +34,11 @@ #include "XrdOss/XrdOss.hh" #include "XrdOssCsiTagstore.hh" #include "XrdOuc/XrdOucCRC.hh" +#include "XrdSys/XrdSysPlatform.hh" #include #include -#if defined(__APPLE__) -// Make sure that byte swap functions are not already defined. -#if !defined(bswap_16) -// Mac OS X / Darwin features -#include -#define bswap_16(x) OSSwapInt16(x) -#define bswap_32(x) OSSwapInt32(x) -#define bswap_64(x) OSSwapInt64(x) -#endif -#else -#include -#endif - class XrdOssCsiTagstoreFile : public XrdOssCsiTagstore { public: diff --git a/src/XrdOssCsi/XrdOssCsiTrace.hh b/src/XrdOssCsi/XrdOssCsiTrace.hh index dc708615cf8..7abf4aaf180 100644 --- a/src/XrdOssCsi/XrdOssCsiTrace.hh +++ b/src/XrdOssCsi/XrdOssCsiTrace.hh @@ -48,13 +48,13 @@ #define TRACE(act, x) \ if (QTRACE(act)) \ - {OssCsiTrace.Beg(epname,tident); cerr < The message routing object to be used in conjunction //! with an XrdSysError object for error messages. When -//! nil, you should use cerr. +//! nil, you should use std::cerr. //! @param Config -> The name of the config file. When nil there was no //! configuration file. //! @param Parms -> Any parameters specified after the path on the diff --git a/src/XrdOuc/XrdOucFileInfo.hh b/src/XrdOuc/XrdOucFileInfo.hh index 5897c3be4b5..3dee49d1851 100644 --- a/src/XrdOuc/XrdOucFileInfo.hh +++ b/src/XrdOuc/XrdOucFileInfo.hh @@ -145,7 +145,7 @@ long long GetSize() {return fSize;} //! //! @return Pointer to the url. The url is valid until this object is deleted. //! If no more urls exist, a nil pointer is returned. A subsequent call -//! will start at the front of teh list. +//! will start at the front of the list. //----------------------------------------------------------------------------- const char *GetUrl(char *cntry=0, int *prty=0); diff --git a/src/XrdOuc/XrdOucGMap.cc b/src/XrdOuc/XrdOucGMap.cc index f576a55cc67..eaa11df5af0 100644 --- a/src/XrdOuc/XrdOucGMap.cc +++ b/src/XrdOuc/XrdOucGMap.cc @@ -57,8 +57,8 @@ enum XrdOucGMap_Match {kFull = 0, kContains = 4 }; -#define PRINT(t,n,y) {if (t) {t->Beg(n); cerr <End();}} -#define DEBUG(d,t,n,y) {if (d && t) {t->Beg(n); cerr <End();}} +#define PRINT(t,n,y) {if (t) {t->Beg(n); std::cerr <End();}} +#define DEBUG(d,t,n,y) {if (d && t) {t->Beg(n); std::cerr <End();}} //__________________________________________________________________________ static int FindMatchingCondition(const char *, XrdSecGMapEntry_t *mc, void *xmp) diff --git a/src/XrdOuc/XrdOucGatherConf.cc b/src/XrdOuc/XrdOucGatherConf.cc index 29179744a4c..cef0c6c2c53 100644 --- a/src/XrdOuc/XrdOucGatherConf.cc +++ b/src/XrdOuc/XrdOucGatherConf.cc @@ -143,7 +143,7 @@ int XrdOucGatherConf::Gather(const char *cfname, Level lvl, const char *parms) while((var = Config.GetMyFirstWord())) {tP = Match; while(tP && ((tP->val && strncmp(var, tP->text, tP->val)) || - strcmp( var, tP->text))) tP = tP->next; + (!tP->val && strcmp( var, tP->text)))) tP = tP->next; if (tP) {if (addKey) diff --git a/src/XrdOuc/XrdOucNSWalk.cc b/src/XrdOuc/XrdOucNSWalk.cc index d93fa7aeb1d..34ca4accdd9 100644 --- a/src/XrdOuc/XrdOucNSWalk.cc +++ b/src/XrdOuc/XrdOucNSWalk.cc @@ -247,9 +247,9 @@ int XrdOucNSWalk::Emsg(const char *pfx, int rc, const char *txt1, if (eDest) eDest->Emsg(pfx, rc, txt1, txt2); else if (mPfx) {const char *etxt = XrdSysE2T(rc); - cerr <:") for use by command line commands. // void setMsgOn(const char *pfx) {mPfx = pfx;} diff --git a/src/XrdOuc/XrdOucPList.hh b/src/XrdOuc/XrdOucPList.hh index 24ff1104f0a..69082ced35c 100644 --- a/src/XrdOuc/XrdOucPList.hh +++ b/src/XrdOuc/XrdOucPList.hh @@ -31,7 +31,7 @@ /******************************************************************************/ #include -#include +#include #include class XrdOucPList diff --git a/src/XrdOuc/XrdOucPinObject.hh b/src/XrdOuc/XrdOucPinObject.hh index e92eafc6728..1bc893718f5 100644 --- a/src/XrdOuc/XrdOucPinObject.hh +++ b/src/XrdOuc/XrdOucPinObject.hh @@ -36,7 +36,7 @@ */ class XrdOucEnv; -class XrdOucLogger; +class XrdSysLogger; template class XrdOucPinObject diff --git a/src/XrdOuc/XrdOucProg.cc b/src/XrdOuc/XrdOucProg.cc index 9a258d80457..39a552932ff 100644 --- a/src/XrdOuc/XrdOucProg.cc +++ b/src/XrdOuc/XrdOucProg.cc @@ -313,7 +313,7 @@ int XrdOucProg::Setup(const char *prog, XrdSysError *errP, if (rc <= 0) {if (errP) {if (!rc || !argV[0]) - {const char *pgm = (Proc ? "proceedure" : "program"); + {const char *pgm = (Proc ? "procedure" : "program"); errP->Emsg("Run", pgm, "name not specified."); } else errP->Emsg("Run", rc, "set up", argV[0]); } @@ -321,7 +321,7 @@ int XrdOucProg::Setup(const char *prog, XrdSysError *errP, } // Record the arguments including the phamtom null pointer. We must have -// atleast one, the program or proceedure name. +// atleast one, the program or procedure name. // numArgs = rc; Arg = new char*[rc+1]; diff --git a/src/XrdOuc/XrdOucPsx.cc b/src/XrdOuc/XrdOucPsx.cc index 4d4b59bf7e3..1af710ddb20 100644 --- a/src/XrdOuc/XrdOucPsx.cc +++ b/src/XrdOuc/XrdOucPsx.cc @@ -715,7 +715,7 @@ bool XrdOucPsx::ParseSet(XrdSysError *Eroute, XrdOucStream &Config) {"DataServerTTL", "DataServerTTL",1}, // Default 300 {"LBServerConn_ttl", "LoadBalancerTTL",1}, // Default 1200 {"LoadBalancerTTL", "LoadBalancerTTL",1}, // Default 1200 - {"ParallelEvtLoop", "ParallelEvtLoop",0}, // Default 3 + {"ParallelEvtLoop", "ParallelEvtLoop",0}, // Default 10 {"ParStreamsPerPhyConn", "SubStreamsPerChannel",0},// Default 1 {"ReadAheadSize", 0,0}, {"ReadAheadStrategy", 0,0}, diff --git a/src/XrdOuc/XrdOucPup.cc b/src/XrdOuc/XrdOucPup.cc index d60395f9b58..cef05b6bdec 100644 --- a/src/XrdOuc/XrdOucPup.cc +++ b/src/XrdOuc/XrdOucPup.cc @@ -157,8 +157,8 @@ int XrdOucPup::Pack(struct iovec *iovP, struct iovec *iovE, XrdOucPupArgs *pup, Dtype = pP->Dtype; do {Base.B08 = (char **)(base + pP->Doffs); - //cerr <<"arg " <NList[pP->Name] ? Names->NList[pP->Name] : "?") <NList[pP->Name] ? Names->NList[pP->Name] : "?") <iov_base = Nil; vP->iov_len = 2; diff --git a/src/XrdOuc/XrdOucStream.cc b/src/XrdOuc/XrdOucStream.cc index 59f6802d349..d12e915d452 100644 --- a/src/XrdOuc/XrdOucStream.cc +++ b/src/XrdOuc/XrdOucStream.cc @@ -83,7 +83,7 @@ // The following is used by child processes prior to exec() to avoid deadlocks // -#define Erx(p, a, b) cerr <<#p <<": " <PutPtr("XrdFSCtl_PC*", pfcFSctl); + return &instance; } } diff --git a/src/XrdPfc/XrdPfcFSctl.cc b/src/XrdPfc/XrdPfcFSctl.cc new file mode 100644 index 00000000000..1a2f1f29952 --- /dev/null +++ b/src/XrdPfc/XrdPfcFSctl.cc @@ -0,0 +1,138 @@ +/******************************************************************************/ +/* */ +/* X r d P f c F S c t l . c c */ +/* */ +/* (c) 2023 by the Board of Trustees of the Leland Stanford, Jr., University */ +/* All Rights Reserved */ +/* Produced by Andrew Hanushevsky for Stanford University under contract */ +/* DE-AC02-76-SFO0515 with the Department of Energy */ +/* */ +/* This file is part of the XRootD software suite. */ +/* */ +/* XRootD is free software: you can redistribute it and/or modify it under */ +/* the terms of the GNU Lesser General Public License as published by the */ +/* Free Software Foundation, either version 3 of the License, or (at your */ +/* option) any later version. */ +/* */ +/* XRootD is distributed in the hope that it will be useful, but WITHOUT */ +/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */ +/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */ +/* License for more details. */ +/* */ +/* You should have received a copy of the GNU Lesser General Public License */ +/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */ +/* COPYING (GPL license). If not, see . */ +/* */ +/* The copyright holder's institutional names and contributor's names may not */ +/* be used to endorse or promote products derived from this software without */ +/* specific prior written permission of the institution or contributor. */ +/******************************************************************************/ + +#include +#include +#include + +#include "XrdOfs/XrdOfsHandle.hh" +#include "XrdOuc/XrdOucEnv.hh" +#include "XrdOuc/XrdOucErrInfo.hh" +#include "XrdPfc/XrdPfc.hh" +#include "XrdPfc/XrdPfcFSctl.hh" +#include "XrdPfc/XrdPfcTrace.hh" +#include "XrdSfs/XrdSfsInterface.hh" +#include "XrdSys/XrdSysTrace.hh" + +/******************************************************************************/ +/* C o n s t r u c t o r */ +/******************************************************************************/ + +XrdPfcFSctl::XrdPfcFSctl(XrdPfc::Cache &cInst, XrdSysLogger *logP) + : myCache(cInst), hProc(0), Log(logP, "PfcFsctl"), + sysTrace(cInst.GetTrace()), m_traceID("PfcFSctl") {} + +/******************************************************************************/ +/* C o n f i g u r e */ +/******************************************************************************/ + +bool XrdPfcFSctl::Configure(const char *CfgFN, + const char *Parms, + XrdOucEnv *envP, + const Plugins &plugs) +{ +// All we are interested in is getting the file handle handler pointer +// + hProc = (XrdOfsHandle*)envP->GetPtr("XrdOfsHandle*"); + return hProc != 0; +} + +/******************************************************************************/ +/* F S c t l [ F i l e ] */ +/******************************************************************************/ + +int XrdPfcFSctl::FSctl(const int cmd, + int alen, + const char *args, + XrdSfsFile &file, + XrdOucErrInfo &eInfo, + const XrdSecEntity *client) +{ + eInfo.setErrInfo(ENOTSUP, "File based fstcl not supported for a cache."); + return SFS_ERROR; +} + +/******************************************************************************/ +/* F S c t l [ B a s e ] */ +/******************************************************************************/ + +int XrdPfcFSctl::FSctl(const int cmd, + XrdSfsFSctl &args, + XrdOucErrInfo &eInfo, + const XrdSecEntity *client) +{ + const char *msg = "", *xeq = args.Arg1; + int ec, rc; + +// Verify command +// + if (cmd != SFS_FSCTL_PLUGXC) + {eInfo.setErrInfo(EIDRM, "None-cache command issued to a cache."); + return SFS_ERROR; + } + +// Very that we have a command +// + if (!xeq || args.Arg1Len < 1) + {eInfo.setErrInfo(EINVAL, "Missing cache command or argument."); + return SFS_ERROR; + } + +// Process command +// + if ((!strcmp(xeq, "evict") || !strcmp(xeq, "fevict")) && args.Arg2Len == -2) + {std::string path = args.ArgP[0]; + ec = myCache.UnlinkFile(path, *xeq != 'f'); + switch(ec) + {case 0: if (hProc) hProc->Hide(path.c_str()); + [[fallthrough]]; + case -ENOENT: rc = SFS_OK; + break; + case -EBUSY: ec = ENOTTY; + rc = SFS_ERROR; + msg = "file is in use"; + break; + case -EAGAIN: rc = 5; + break; + default: rc = SFS_ERROR; + msg = "unlink failed"; + break; + } + TRACE(Info,"Cache "<_flags |= _IO_ERR_SEEN; -#elif defined __sferror || defined __APPLE__ || defined __DragonFly__ || defined __ANDROID__ +#elif defined __sferror || defined __APPLE__ || defined __DragonFly__ || defined __FreeBSD__ || defined __ANDROID__ /* FreeBSD, NetBSD, OpenBSD, DragonFly, Mac OS X, Cygwin, Minix 3, Android */ fp->_flags |= __SERR; #elif defined _IOERR diff --git a/src/XrdPosix/XrdPosixAdmin.cc b/src/XrdPosix/XrdPosixAdmin.cc index 3b7e9810353..5e37e72a267 100644 --- a/src/XrdPosix/XrdPosixAdmin.cc +++ b/src/XrdPosix/XrdPosixAdmin.cc @@ -51,7 +51,7 @@ XrdCl::URL *XrdPosixAdmin::FanOut(int &num) XrdCl::URL *uVec; XrdNetAddr netLoc; const char *hName; - unsigned int i; + unsigned long i; // Make sure admin is ok // diff --git a/src/XrdPosix/XrdPosixConfig.cc b/src/XrdPosix/XrdPosixConfig.cc index 9a20c99ac36..e08c651971c 100644 --- a/src/XrdPosix/XrdPosixConfig.cc +++ b/src/XrdPosix/XrdPosixConfig.cc @@ -488,7 +488,7 @@ bool XrdPosixConfig::SetConfig(XrdOucPsx &parms) void XrdPosixConfig::SetDebug(int val) { - const std::string dbgType[] = {"Info", "Warning", "Error", "Debug", "Dump"}; + const std::string dbgType[] = {"Error", "Warning", "Info", "Debug", "Dump"}; // The default is none but once set it cannot be unset in the client // diff --git a/src/XrdPosix/XrdPosixFile.hh b/src/XrdPosix/XrdPosixFile.hh index 4ca9679f233..6d99cf46489 100644 --- a/src/XrdPosix/XrdPosixFile.hh +++ b/src/XrdPosix/XrdPosixFile.hh @@ -166,7 +166,7 @@ inline void UpdtSize(size_t newsz) using XrdPosixObject::Who; -inline bool Who(XrdPosixFile **fileP) +inline bool Who(XrdPosixFile **fileP) override {*fileP = this; return true;} int Write(char *Buff, long long Offs, int Len) override; diff --git a/src/XrdPosix/XrdPosixLinkage.cc b/src/XrdPosix/XrdPosixLinkage.cc index bf6aab82e42..6115b69122f 100644 --- a/src/XrdPosix/XrdPosixLinkage.cc +++ b/src/XrdPosix/XrdPosixLinkage.cc @@ -276,7 +276,7 @@ int XrdPosixLinkage::Resolve() int XrdPosixLinkage::Load_Error(const char *epname, int retv) { if (*Write != &Xrd_U_Write && *Writev != &Xrd_U_Writev) - cerr << "PosixPreload: Unable to resolve Unix '" <What <<"()'" <What <<"()'" <Next; } } diff --git a/src/XrdPosix/XrdPosixMap.cc b/src/XrdPosix/XrdPosixMap.cc index a5a06230a06..7796c926e50 100644 --- a/src/XrdPosix/XrdPosixMap.cc +++ b/src/XrdPosix/XrdPosixMap.cc @@ -169,7 +169,7 @@ int XrdPosixMap::Result(const XrdCl::XRootDStatus &Status, bool retneg1) // make this messae useful like the opteration and path). // // if (eNum != ENOENT && !eText.empty() && Debug) -// cerr <<"XrdPosix: " < - #include "XrdCl/XrdClFileSystem.hh" #include "XrdCl/XrdClXRootDResponses.hh" +#include +#include + class XrdPosixMap { public: diff --git a/src/XrdPosix/XrdPosixPreload.cc b/src/XrdPosix/XrdPosixPreload.cc index e01bc8a2301..8db1400d485 100644 --- a/src/XrdPosix/XrdPosixPreload.cc +++ b/src/XrdPosix/XrdPosixPreload.cc @@ -28,6 +28,10 @@ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ +#if defined(__clang__) && defined(_FORTIFY_SOURCE) +#undef _FORTIFY_SOURCE +#endif + #include #include #include diff --git a/src/XrdPosix/XrdPosixPreload32.cc b/src/XrdPosix/XrdPosixPreload32.cc index 7280dd2ec13..3d99ba9d2ee 100644 --- a/src/XrdPosix/XrdPosixPreload32.cc +++ b/src/XrdPosix/XrdPosixPreload32.cc @@ -28,6 +28,11 @@ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ +#if defined(__clang__) && defined(_FORTIFY_SOURCE) +#undef _FORTIFY_SOURCE +#endif + +#if !defined(MUSL) #ifdef _LARGEFILE_SOURCE #undef _LARGEFILE_SOURCE #endif @@ -39,6 +44,7 @@ #ifdef _FILE_OFFSET_BITS #undef _FILE_OFFSET_BITS #endif +#endif #define XRDPOSIXPRELOAD32 diff --git a/src/XrdPosix/XrdPosixXrootdPath.hh b/src/XrdPosix/XrdPosixXrootdPath.hh index 8a9c2fc0c20..7bab33b6e40 100644 --- a/src/XrdPosix/XrdPosixXrootdPath.hh +++ b/src/XrdPosix/XrdPosixXrootdPath.hh @@ -29,6 +29,8 @@ /* be used to endorse or promote products derived from this software without */ /* specific prior written permission of the institution or contributor. */ /******************************************************************************/ + +#include class XrdPosixXrootPath { diff --git a/src/XrdPss/XrdPss.hh b/src/XrdPss/XrdPss.hh index 798312a727e..631013ea6f5 100644 --- a/src/XrdPss/XrdPss.hh +++ b/src/XrdPss/XrdPss.hh @@ -144,34 +144,34 @@ struct XrdVersionInfo; class XrdPssSys : public XrdOss { public: -virtual XrdOssDF *newDir(const char *tident) +virtual XrdOssDF *newDir(const char *tident) override {return (XrdOssDF *)new XrdPssDir(tident);} -virtual XrdOssDF *newFile(const char *tident) +virtual XrdOssDF *newFile(const char *tident) override {return (XrdOssDF *)new XrdPssFile(tident);} -virtual void Connect(XrdOucEnv &); +virtual void Connect(XrdOucEnv &) override; -virtual void Disc(XrdOucEnv &); +virtual void Disc(XrdOucEnv &) override; -int Chmod(const char *, mode_t mode, XrdOucEnv *eP=0); +int Chmod(const char *, mode_t mode, XrdOucEnv *eP=0) override; bool ConfigMapID(); virtual -int Create(const char *, const char *, mode_t, XrdOucEnv &, int opts=0); -void EnvInfo(XrdOucEnv *envP); -uint64_t Features() {return myFeatures;} +int Create(const char *, const char *, mode_t, XrdOucEnv &, int opts=0) override; +void EnvInfo(XrdOucEnv *envP) override; +uint64_t Features() override {return myFeatures;} int Init(XrdSysLogger *, const char *) override {return -ENOTSUP;} int Init(XrdSysLogger *, const char *, XrdOucEnv *envP) override; -int Lfn2Pfn(const char *Path, char *buff, int blen); +int Lfn2Pfn(const char *Path, char *buff, int blen) override; const -char *Lfn2Pfn(const char *Path, char *buff, int blen, int &rc); -int Mkdir(const char *, mode_t mode, int mkpath=0, XrdOucEnv *eP=0); -int Remdir(const char *, int Opts=0, XrdOucEnv *eP=0); +char *Lfn2Pfn(const char *Path, char *buff, int blen, int &rc) override; +int Mkdir(const char *, mode_t mode, int mkpath=0, XrdOucEnv *eP=0) override; +int Remdir(const char *, int Opts=0, XrdOucEnv *eP=0) override; int Rename(const char *, const char *, - XrdOucEnv *eP1=0, XrdOucEnv *eP2=0); -int Stat(const char *, struct stat *, int opts=0, XrdOucEnv *eP=0); -int Stats(char *bp, int bl); -int Truncate(const char *, unsigned long long, XrdOucEnv *eP=0); -int Unlink(const char *, int Opts=0, XrdOucEnv *eP=0); + XrdOucEnv *eP1=0, XrdOucEnv *eP2=0) override; +int Stat(const char *, struct stat *, int opts=0, XrdOucEnv *eP=0) override; +int Stats(char *bp, int bl) override; +int Truncate(const char *, unsigned long long, XrdOucEnv *eP=0) override; +int Unlink(const char *, int Opts=0, XrdOucEnv *eP=0) override; static const int PolNum = 2; enum PolAct {PolPath = 0, PolObj = 1}; diff --git a/src/XrdPss/XrdPssConfig.cc b/src/XrdPss/XrdPssConfig.cc index 5c6fc12338e..7295706aa73 100644 --- a/src/XrdPss/XrdPssConfig.cc +++ b/src/XrdPss/XrdPssConfig.cc @@ -199,7 +199,7 @@ int XrdPssSys::Configure(const char *cfn, XrdOucEnv *envP) // Set default number of event loops // - XrdPosixConfig::SetEnv("ParallelEvtLoop", 3); + XrdPosixConfig::SetEnv("ParallelEvtLoop", 10); // Turn off the fork handler as we always exec after forking. // @@ -782,7 +782,7 @@ int XrdPssSys::xorig(XrdSysError *errp, XrdOucStream &Config) // Check if there is a port number. This could be as ':port' or ' port'. // if (!(val = index(mval,':')) && !isURL) val = Config.GetWord(); - else {*val = '\0'; val++;} + else if (val) {*val = '\0'; val++;} // At this point, make sure we actually have a host name // @@ -800,7 +800,17 @@ int XrdPssSys::xorig(XrdSysError *errp, XrdOucStream &Config) {errp->Emsg("Config", "unable to find tcp service", val); port = 0; } - } else errp->Emsg("Config","origin port not specified for",mval); + } else { + if (protName) { + // use default port for protocol + port = *protName == 'h' ? (strncmp(protName, "https", 5) == 0 ? 443 : 80) : 1094; + } else { + // assume protocol is root(s):// + port = 1094; + } + errp->Say("Config warning: origin port not specified, using port ", + std::to_string(port).c_str(), " as default for ", protName); + } // If port is invalid or missing, fail this // diff --git a/src/XrdRmc/XrdRmcData.cc b/src/XrdRmc/XrdRmcData.cc index ea65d494efe..12f77261860 100644 --- a/src/XrdRmc/XrdRmcData.cc +++ b/src/XrdRmc/XrdRmcData.cc @@ -137,7 +137,7 @@ bool XrdRmcData::Detach(XrdOucCacheIOCD &iocd) Statistics.X.BytesPead, Statistics.X.HitsPR, Statistics.X.MissPR, ioObj->Path()); - cerr <= prMax) prNext = 0; if (oVal == prSKIP) continue; prActive = prRun; - if (Debug > 1) cerr <<"prD: beg " <<(VNum >>XrdRmcReal::Shift) <<' ' + if (Debug > 1) std::cerr <<"prD: beg " <<(VNum >>XrdRmcReal::Shift) <<' ' <<(segEnd-segBeg+1)*SegSize <<'@' <<(segBeg*SegSize) - <<" f=" <Path() <>XrdRmcReal::Shift) - <<' ' < 1) std::cerr <<"PrD: end " <<(VNum >>XrdRmcReal::Shift) + <<' ' < prBeg[i] && segEnd <= prEnd[i])) {if (prHow == prSKIP) - {if (Debug) cerr <<"pDQ: " <Path() <Path() <= 0) {if ( crPerf < Apr.minPerf && prPerf < Apr.minPerf && (crPerf <= prPerf || crPerf <= prPerf*2)) - {if (Debug) cerr <<"PrD: Disabled for " <Path() <Path() <PreRead(&prReq);} } @@ -376,7 +376,7 @@ int XrdRmcData::Read(char *Buff, long long Offs, int rLen) DMutex.UnLock(); } } - if (Debug > 1) cerr <<"Rdr: " < 1) std::cerr <<"Rdr: ret " <<(cBuff ? Dest-Buff : rGot) <<" hits " + < rLen) rAmt = rLen; - if (Debug > 1) cerr <<"Rdr: " < 1) std::cerr <<"Rdr: ret " <<(Dest-Buff) <<" hits " <Path() <Path() <Path() <Path() < 1) std::cerr <<"Cache: Wait slot " <Contents != lAddr) {rAmt = -EIO; return 0;} } else { @@ -335,8 +335,8 @@ char *XrdRmcReal::Get(XrdOucCacheIO *ioP, long long lAddr, int &rAmt, int &noIO) rAmt = (sP->Count < 0 ? sP->Count & XrdRmcSlot::lenMask : SegSize); if (sP->Count & XrdRmcSlot::isNew) {noIO = -1; sP->Count &= ~XrdRmcSlot::isNew;} - if (Dbg > 2) cerr <<"Cache: Hit slot " <Status.inUse < 2) std::cerr <<"Cache: Hit slot " <Status.inUse <(Slot)*SegSize); } @@ -384,8 +384,8 @@ char *XrdRmcReal::Get(XrdOucCacheIO *ioP, long long lAddr, int &rAmt, int &noIO) Slots[Fnum].Owner(Slots, sP); sP->Count = (rAmt == SegSize ? SegFull : rAmt|XrdRmcSlot::isShort); sP->Status.inUse = nUse; - if (Dbg > 2) cerr <<"Cache: Miss slot " <Count & XrdRmcSlot::lenMask) < 2) std::cerr <<"Cache: Miss slot " <Count & XrdRmcSlot::lenMask) <Path(), "reading", (lAddr & Strip) << SegShft, SegSize, rAmt); cBuff = 0; @@ -459,7 +459,7 @@ void XrdRmcReal::PreRead() // Simply wait and dispatch elements // - if (Dbg) cerr <<"Cache: preread thread started; now " < 0) prReady.Post(); else prStop->Post(); - if (Dbg) cerr <<"Cache: preread thread exited; left " < 2) cerr <<"Cache: Ref " <Contents < 2) std::cerr <<"Cache: Ref " <Contents <>SegShft) <<" sz " <<(sP->Count & XrdRmcSlot::lenMask) - <<" uc " <Status.inUse <Status.inUse <Path() <Path() < 2) cerr <<"Cache: Upd " <Contents < 2) std::cerr <<"Cache: Upd " <Contents <>SegShft) <<" sz " <<(sP->Count & XrdRmcSlot::lenMask) - <<" uc " <Status.inUse <Status.inUse < @@ -24,6 +25,7 @@ #include "scitokens/scitokens.h" #include "XrdSciTokens/XrdSciTokensHelper.hh" +#include "XrdSciTokens/XrdSciTokensMon.hh" // The status-quo to retrieve the default object is to copy/paste the // linker definition and invoke directly. @@ -218,23 +220,28 @@ void ParseCanonicalPaths(const std::string &path, std::vector &resu struct MapRule { MapRule(const std::string &sub, + const std::string &username, const std::string &path_prefix, const std::string &group, - const std::string &name) + const std::string &result) : m_sub(sub), + m_username(username), m_path_prefix(path_prefix), m_group(group), - m_name(name) + m_result(result) { - //std::cerr << "Making a rule {sub=" << sub << ", path=" << path_prefix << ", group=" << group << ", result=" << name << "}" << std::endl; + //std::cerr << "Making a rule {sub=" << sub << ", username=" << username << ", path=" << path_prefix << ", group=" << group << ", result=" << name << "}" << std::endl; } - const std::string match(const std::string sub, - const std::string req_path, - const std::vector groups) const + const std::string match(const std::string &sub, + const std::string &username, + const std::string &req_path, + const std::vector &groups) const { if (!m_sub.empty() && sub != m_sub) {return "";} + if (!m_username.empty() && username != m_username) {return "";} + if (!m_path_prefix.empty() && strncmp(req_path.c_str(), m_path_prefix.c_str(), m_path_prefix.size())) { @@ -244,17 +251,18 @@ struct MapRule if (!m_group.empty()) { for (const auto &group : groups) { if (group == m_group) - return m_name; + return m_result; } return ""; } - return m_name; + return m_result; } std::string m_sub; + std::string m_username; std::string m_path_prefix; std::string m_group; - std::string m_name; + std::string m_result; }; struct IssuerConfig @@ -265,11 +273,13 @@ struct IssuerConfig const std::vector &restricted_paths, bool map_subject, const std::string &default_user, + const std::string &username_claim, const std::vector rules) - : m_map_subject(map_subject), + : m_map_subject(map_subject || !username_claim.empty()), m_name(issuer_name), m_url(issuer_url), m_default_user(default_user), + m_username_claim(username_claim), m_base_paths(base_paths), m_restricted_paths(restricted_paths), m_map_rules(rules) @@ -279,6 +289,7 @@ struct IssuerConfig const std::string m_name; const std::string m_url; const std::string m_default_user; + const std::string m_username_claim; const std::vector m_base_paths; const std::vector m_restricted_paths; const std::vector m_map_rules; @@ -339,7 +350,23 @@ class XrdAccRules bool apply(Access_Operation oper, std::string path) { for (const auto & rule : m_rules) { - if ((oper == rule.first) && !path.compare(0, rule.second.size(), rule.second, 0, rule.second.size())) { + // The rule permits if both conditions are met: + // - The operation type matches the requested operation, + // - The requested path is a substring of the ACL's permitted path, AND + // - Either the requested path and ACL path is the same OR the requested path is a subdir of the ACL path. + // + // The third rule implies if the rule permits read:/foo, we should NOT authorize read:/foobar. + if ((oper == rule.first) && + !path.compare(0, rule.second.size(), rule.second, 0, rule.second.size()) && + (rule.second.size() == path.length() || path[rule.second.size()]=='/')) + { + return true; + } + // according to WLCG token specs, allow creation of required superfolders for a new file if requested + if ((oper == rule.first) && (oper == AOP_Stat || oper == AOP_Mkdir) + && rule.second.size() >= path.length() + && !rule.second.compare(0, path.size(), path, 0, path.size()) + && (rule.second.size() == path.length() || rule.second[path.length()] == '/')) { return true; } } @@ -358,7 +385,7 @@ class XrdAccRules std::string get_username(const std::string &req_path) const { for (const auto &rule : m_map_rules) { - std::string name = rule.match(m_token_subject, req_path, m_groups); + std::string name = rule.match(m_token_subject, m_username, req_path, m_groups); if (!name.empty()) { return name; } @@ -411,7 +438,8 @@ class XrdAccSciTokens; XrdAccSciTokens *accSciTokens = nullptr; XrdSciTokensHelper *SciTokensHelper = nullptr; -class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper +class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper, + public XrdSciTokensMon { enum class AuthzBehavior { @@ -421,7 +449,7 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper }; public: - XrdAccSciTokens(XrdSysLogger *lp, const char *parms, XrdAccAuthorize* chain) : + XrdAccSciTokens(XrdSysLogger *lp, const char *parms, XrdAccAuthorize* chain, XrdOucEnv *envP) : m_chain(chain), m_parms(parms ? parms : ""), m_next_clean(monotonic_time() + m_expiry_secs), @@ -430,7 +458,7 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper pthread_rwlock_init(&m_config_lock, nullptr); m_config_lock_initialized = true; m_log.Say("++++++ XrdAccSciTokens: Initialized SciTokens-based authorization."); - if (!Config()) { + if (!Config(envP)) { throw std::runtime_error("Failed to configure SciTokens authorization."); } } @@ -519,6 +547,8 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper new_secentity.vorg = nullptr; new_secentity.grps = nullptr; new_secentity.role = nullptr; + new_secentity.secMon = Entity->secMon; + new_secentity.addrInfo = Entity->addrInfo; const auto &issuer = access_rules->get_issuer(); if (!issuer.empty()) { new_secentity.vorg = strdup(issuer.c_str()); @@ -584,6 +614,12 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper // When the scope authorized this access, allow immediately. Otherwise, chain XrdAccPrivs returned_op = scope_success ? AddPriv(oper, XrdAccPriv_None) : OnMissing(&new_secentity, path, oper, env); + // Since we are doing an early return, insert token info into the + // monitoring stream if monitoring is in effect and access granted + // + if (Entity->secMon && scope_success && returned_op && Mon_isIO(oper)) + Mon_Report(new_secentity, token_subject, username); + // Cleanup the new_secentry if (new_secentity.vorg != nullptr) free(new_secentity.vorg); if (new_secentity.grps != nullptr) free(new_secentity.grps); @@ -614,7 +650,7 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper } virtual bool Validate(const char *token, std::string &emsg, long long *expT, - XrdSecEntity *Entity) + XrdSecEntity *Entity) override { // Just check if the token is valid, no scope checking @@ -811,10 +847,18 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper token_subject = std::string(value); free(value); - std::string tmp_username; - if (config.m_map_subject) { - tmp_username = token_subject; - } else { + auto tmp_username = token_subject; + if (!config.m_username_claim.empty()) { + if (scitoken_get_claim_string(token, config.m_username_claim.c_str(), &value, &err_msg)) { + pthread_rwlock_unlock(&m_config_lock); + m_log.Log(LogMask::Warning, "GenerateAcls", "Failed to get token username:", err_msg); + free(err_msg); + scitoken_destroy(token); + return false; + } + tmp_username = std::string(value); + free(value); + } else if (!config.m_map_subject) { tmp_username = config.m_default_user; } @@ -834,20 +878,47 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper int idx = 0; std::set paths_write_seen; std::set paths_create_or_modify_seen; + std::vector acl_paths; + acl_paths.reserve(config.m_restricted_paths.size() + 1); while (acls[idx].resource && acls[idx++].authz) { + acl_paths.clear(); const auto &acl_path = acls[idx-1].resource; const auto &acl_authz = acls[idx-1].authz; - if (!config.m_restricted_paths.empty()) { - bool found_path = false; + if (config.m_restricted_paths.empty()) { + acl_paths.push_back(acl_path); + } else { + auto acl_path_size = strlen(acl_path); for (const auto &restricted_path : config.m_restricted_paths) { + // See if the acl_path is more specific than the restricted path; if so, accept it + // and move on to applying paths. if (!strncmp(acl_path, restricted_path.c_str(), restricted_path.size())) { - found_path = true; + // Only do prefix checking on full path components. If acl_path=/foobar and + // restricted_path=/foo, then we shouldn't authorize access to /foobar. + if (acl_path_size > restricted_path.size() && acl_path[restricted_path.size()] != '/') { + continue; + } + acl_paths.push_back(acl_path); break; } + // See if the restricted_path is more specific than the acl_path; if so, accept the + // restricted path as the ACL. Keep looping to see if other restricted paths add + // more possible authorizations. + if (!strncmp(acl_path, restricted_path.c_str(), acl_path_size)) { + // Only do prefix checking on full path components. If acl_path=/foo and + // restricted_path=/foobar, then we shouldn't authorize access to /foobar. Note: + // - The scitokens-cpp library guaranteees that acl_path is normalized and not + // of the form `/foo/`. + // - Hence, the only time that the acl_path can end in a '/' is when it is + // set to `/`. + if ((restricted_path.size() > acl_path_size && restricted_path[acl_path_size] != '/') && (acl_path_size != 1)) { + continue; + } + acl_paths.push_back(restricted_path); + } } - if (!found_path) {continue;} } - for (const auto &base_path : config.m_base_paths) { + for (const auto &acl_path : acl_paths) { + for (const auto &base_path : config.m_base_paths) { if (!acl_path[0] || acl_path[0] != '/') {continue;} std::string path; MakeCanonical(base_path + acl_path, path); @@ -861,6 +932,7 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper xrd_rules.emplace_back(AOP_Mkdir, path); xrd_rules.emplace_back(AOP_Rename, path); xrd_rules.emplace_back(AOP_Excl_Insert, path); + xrd_rules.emplace_back(AOP_Stat, path); } else if (!strcmp(acl_authz, "modify")) { paths_create_or_modify_seen.insert(path); xrd_rules.emplace_back(AOP_Create, path); @@ -869,10 +941,12 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper xrd_rules.emplace_back(AOP_Insert, path); xrd_rules.emplace_back(AOP_Update, path); xrd_rules.emplace_back(AOP_Chmod, path); + xrd_rules.emplace_back(AOP_Stat, path); xrd_rules.emplace_back(AOP_Delete, path); } else if (!strcmp(acl_authz, "write")) { paths_write_seen.insert(path); } + } } } for (const auto &write_path : paths_write_seen) { @@ -883,6 +957,7 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper xrd_rules.emplace_back(AOP_Rename, write_path); xrd_rules.emplace_back(AOP_Insert, write_path); xrd_rules.emplace_back(AOP_Update, write_path); + xrd_rules.emplace_back(AOP_Stat, write_path); xrd_rules.emplace_back(AOP_Chmod, write_path); xrd_rules.emplace_back(AOP_Delete, write_path); } @@ -900,7 +975,7 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper } - bool Config() { + bool Config(XrdOucEnv *envP) { // Set default mask for logging. m_log.setMsgMask(LogMask::Error | LogMask::Warning); @@ -936,6 +1011,19 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper } m_log.Emsg("Config", "Logging levels enabled -", LogMaskToString(m_log.getMsgMask()).c_str()); + auto xrdEnv = static_cast(envP ? envP->GetPtr("xrdEnv*") : nullptr); + auto tlsCtx = static_cast(xrdEnv ? xrdEnv->GetPtr("XrdTlsContext*") : nullptr); + if (tlsCtx) { + auto params = tlsCtx->GetParams(); + if (params && !params->cafile.empty()) { +#ifdef HAVE_SCITOKEN_CONFIG_SET_STR + scitoken_config_set_str("tls.ca_file", params->cafile.c_str(), nullptr); +#else + m_log.Log(LogMask::Warning, "Config", "tls.ca_file is set but the platform's libscitokens.so does not support setting config parameters"); +#endif + } + } + return Reconfig(); } @@ -972,6 +1060,7 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper std::string path; std::string group; std::string sub; + std::string username; std::string result; bool ignore = false; for (const auto &entry : rule.get()) { @@ -989,8 +1078,9 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper } else if (entry.first == "sub") { sub = entry.second.get(); - } - else if (entry.first == "path") { + } else if (entry.first == "username") { + username = entry.second.get(); + } else if (entry.first == "path") { std::string norm_path; if (!MakeCanonical(entry.second.get(), norm_path)) { ss << "In mapfile " << filename << " encountered a path " << entry.second.get() @@ -1011,7 +1101,7 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper m_log.Log(LogMask::Error, "ParseMapfile", ss.str().c_str()); return false; } - rules.emplace_back(sub, path, group, result); + rules.emplace_back(sub, username, path, group, result); } return true; @@ -1158,11 +1248,12 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper auto default_user = reader.Get(section, "default_user", ""); auto map_subject = reader.GetBoolean(section, "map_subject", false); + auto username_claim = reader.Get(section, "username_claim", ""); issuers.emplace(std::piecewise_construct, std::forward_as_tuple(issuer), std::forward_as_tuple(name, issuer, base_paths, restricted_paths, - map_subject, default_user, rules)); + map_subject, default_user, username_claim, rules)); } if (issuers.empty()) { @@ -1230,10 +1321,10 @@ class XrdAccSciTokens : public XrdAccAuthorize, public XrdSciTokensHelper }; void InitAccSciTokens(XrdSysLogger *lp, const char *cfn, const char *parm, - XrdAccAuthorize *accP) + XrdAccAuthorize *accP, XrdOucEnv *envP) { try { - accSciTokens = new XrdAccSciTokens(lp, parm, accP); + accSciTokens = new XrdAccSciTokens(lp, parm, accP, envP); SciTokensHelper = accSciTokens; } catch (std::exception &) { } @@ -1244,7 +1335,7 @@ extern "C" { XrdAccAuthorize *XrdAccAuthorizeObjAdd(XrdSysLogger *lp, const char *cfn, const char *parm, - XrdOucEnv * /*not used*/, + XrdOucEnv *envP, XrdAccAuthorize *accP) { // Record the parent authorization plugin. There is no need to use @@ -1254,7 +1345,7 @@ XrdAccAuthorize *XrdAccAuthorizeObjAdd(XrdSysLogger *lp, // If we have been initialized by a previous load, them return that result. // Otherwise, it's the first time through, get a new SciTokens authorizer. // - if (!accSciTokens) InitAccSciTokens(lp, cfn, parm, accP); + if (!accSciTokens) InitAccSciTokens(lp, cfn, parm, accP, envP); return accSciTokens; } @@ -1262,7 +1353,16 @@ XrdAccAuthorize *XrdAccAuthorizeObject(XrdSysLogger *lp, const char *cfn, const char *parm) { - InitAccSciTokens(lp, cfn, parm, 0); + InitAccSciTokens(lp, cfn, parm, nullptr, nullptr); + return accSciTokens; +} + +XrdAccAuthorize *XrdAccAuthorizeObject2(XrdSysLogger *lp, + const char *cfn, + const char *parm, + XrdOucEnv *envP) +{ + InitAccSciTokens(lp, cfn, parm, nullptr, envP); return accSciTokens; } diff --git a/src/XrdSciTokens/XrdSciTokensHelper.hh b/src/XrdSciTokens/XrdSciTokensHelper.hh index bab032d9599..aeac5ddb5ca 100644 --- a/src/XrdSciTokens/XrdSciTokensHelper.hh +++ b/src/XrdSciTokens/XrdSciTokensHelper.hh @@ -1,3 +1,5 @@ +#ifndef __XrdSciTokensHelper_hh__ +#define __XrdSciTokensHelper_hh__ /******************************************************************************/ /* */ /* X r d S c i T o k e n s H e l p e r . h h */ @@ -65,3 +67,4 @@ virtual bool Validate(const char *token, XrdSciTokensHelper() {} virtual ~XrdSciTokensHelper() {} }; +#endif diff --git a/src/XrdSciTokens/XrdSciTokensMon.cc b/src/XrdSciTokens/XrdSciTokensMon.cc new file mode 100644 index 00000000000..df408b07e3e --- /dev/null +++ b/src/XrdSciTokens/XrdSciTokensMon.cc @@ -0,0 +1,31 @@ +/******************************************************************************/ +/* */ +/* X r d S c i T o k e n s M o n . c c */ +/* */ +/******************************************************************************/ + +#include "XrdSciTokens/XrdSciTokensMon.hh" +#include "XrdSec/XrdSecEntity.hh" +#include "XrdSec/XrdSecMonitor.hh" + +/******************************************************************************/ +/* R e p o r t */ +/******************************************************************************/ + +void XrdSciTokensMon::Mon_Report(const XrdSecEntity& Entity, + const std::string& subject, + const std::string& username) +{ +// Create record +// + if (Entity.secMon) + {char buff[2048]; + snprintf(buff, sizeof(buff), + "s=%s&n=%s&o=%s&r=%s&g=%.1024s", + subject.c_str(),username.c_str(), + (Entity.vorg ? Entity.vorg : ""), + (Entity.role ? Entity.role : ""), + (Entity.grps ? Entity.grps : "")); + Entity.secMon->Report(XrdSecMonitor::TokenInfo, buff); + } +} diff --git a/src/XrdSciTokens/XrdSciTokensMon.hh b/src/XrdSciTokens/XrdSciTokensMon.hh new file mode 100644 index 00000000000..6f41787e302 --- /dev/null +++ b/src/XrdSciTokens/XrdSciTokensMon.hh @@ -0,0 +1,28 @@ +#ifndef __XrdSciTokensMon_hh__ +#define __XrdSciTokensMon_hh__ +/******************************************************************************/ +/* */ +/* X r d S c o T o k e n s M o n . h h */ +/* */ +/******************************************************************************/ + +#include + +#include "XrdAcc/XrdAccAuthorize.hh" + +class XrdSciTokensMon +{ +public: + +bool Mon_isIO(const Access_Operation oper) + {return oper == AOP_Read || oper == AOP_Update + || oper == AOP_Create || oper == AOP_Excl_Create; + } + +void Mon_Report(const XrdSecEntity& Entity, const std::string& subject, + const std::string& username); + + XrdSciTokensMon() {} + ~XrdSciTokensMon() {} +}; +#endif diff --git a/src/XrdSciTokens/vendor/inih/INIReader.h b/src/XrdSciTokens/vendor/inih/INIReader.h index 0c7ec07a248..961858e9ec1 100644 --- a/src/XrdSciTokens/vendor/inih/INIReader.h +++ b/src/XrdSciTokens/vendor/inih/INIReader.h @@ -89,7 +89,7 @@ int ini_parse_stream(ini_reader reader, void* stream, ini_handler handler, /* Maximum line length for any line in INI file. */ #ifndef INI_MAX_LINE -#define INI_MAX_LINE 200 +#define INI_MAX_LINE 1024 #endif #ifdef __cplusplus @@ -117,8 +117,8 @@ home page for more info: #include #endif -#define MAX_SECTION 50 -#define MAX_NAME 50 +#define MAX_SECTION 1024 +#define MAX_NAME 1024 /* Strip whitespace chars off end of given string, in place. Return s. */ inline static char* rstrip(char* s) diff --git a/src/XrdSciTokens/vendor/picojson/README.mkdn b/src/XrdSciTokens/vendor/picojson/README.mkdn index c68153011ed..75401b03800 100644 --- a/src/XrdSciTokens/vendor/picojson/README.mkdn +++ b/src/XrdSciTokens/vendor/picojson/README.mkdn @@ -157,7 +157,7 @@ Please refer to the implementation of picojson::default_parse_context and picojs ## Serializing to JSON -Instances of the picojson::value class can be serialized in three ways, to ostream, to std::string, or to an output iterator. +Instances of the picojson::value class can be serialized in three ways, to std::ostream, to std::string, or to an output iterator. 
 picojson::value v;
diff --git a/src/XrdSciTokens/vendor/picojson/examples/github-issues.cc b/src/XrdSciTokens/vendor/picojson/examples/github-issues.cc
index 0235b965783..4bfb15b12ac 100644
--- a/src/XrdSciTokens/vendor/picojson/examples/github-issues.cc
+++ b/src/XrdSciTokens/vendor/picojson/examples/github-issues.cc
@@ -82,7 +82,7 @@ int main(int argc, char *argv[]) {
   curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, memfwrite);
   curl_easy_setopt(curl, CURLOPT_WRITEDATA, mf);
   if (curl_easy_perform(curl) != CURLE_OK) {
-    cerr << error << endl;
+    std::cerr << error << std::endl;
   } else {
     value v;
     string err;
@@ -92,11 +92,11 @@ int main(int argc, char *argv[]) {
       array::iterator it;
       for (it = arr.begin(); it != arr.end(); it++) {
         object obj = it->get();
-        cout << "#" << obj["number"].to_str() << ": " << obj["title"].to_str() << endl;
-        cout << "  " << obj["html_url"].to_str() << endl << endl;
+        std::cout << "#" << obj["number"].to_str() << ": " << obj["title"].to_str() << std::endl;
+        std::cout << "  " << obj["html_url"].to_str() << std::endl << endl;
       }
     } else {
-      cerr << err << endl;
+      std::cerr << err << std::endl;
     }
   }
   curl_easy_cleanup(curl);
diff --git a/src/XrdSec.cmake b/src/XrdSec.cmake
index 0b902f9401d..2e3776a5ecb 100644
--- a/src/XrdSec.cmake
+++ b/src/XrdSec.cmake
@@ -1,5 +1,4 @@
 
-include( XRootDCommon )
 
 #-------------------------------------------------------------------------------
 # Modules
@@ -34,16 +33,11 @@ add_library(
 
 target_link_libraries(
   ${LIB_XRD_SEC}
+  PRIVATE
   XrdUtils
   ${CMAKE_THREAD_LIBS_INIT}
   ${CMAKE_DL_LIBS} )
 
-set_target_properties(
-  ${LIB_XRD_SEC}
-  PROPERTIES
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
-
 #-------------------------------------------------------------------------------
 # The XrdSecpwd module
 #-------------------------------------------------------------------------------
@@ -58,16 +52,11 @@ add_library(
 
 target_link_libraries(
   ${LIB_XRD_SEC_PROT}
+  PRIVATE
   XrdUtils
   ${CMAKE_THREAD_LIBS_INIT}
   OpenSSL::Crypto )
 
-set_target_properties(
-  ${LIB_XRD_SEC_PROT}
-  PROPERTIES
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
-
 #-------------------------------------------------------------------------------
 # The XrdSecpwd module
 #-------------------------------------------------------------------------------
@@ -79,17 +68,12 @@ add_library(
 
 target_link_libraries(
   ${LIB_XRD_SEC_PWD}
+  PRIVATE
   XrdCrypto
   XrdUtils
   ${CMAKE_THREAD_LIBS_INIT}
   ${CRYPT_LIBRARY} )
 
-set_target_properties(
-  ${LIB_XRD_SEC_PWD}
-  PROPERTIES
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
-
 if( NOT XRDCL_LIB_ONLY )
 #-------------------------------------------------------------------------------
 # xrdpwdadmin
@@ -115,15 +99,10 @@ add_library(
 
 target_link_libraries(
   ${LIB_XRD_SEC_SSS}
+  PRIVATE
   XrdCryptoLite
   XrdUtils )
 
-set_target_properties(
-  ${LIB_XRD_SEC_SSS}
-  PROPERTIES
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
-
 if( NOT XRDCL_LIB_ONLY )
 #-------------------------------------------------------------------------------
 # xrdsssadmin
@@ -147,14 +126,9 @@ add_library(
 
 target_link_libraries(
   ${LIB_XRD_SEC_UNIX}
+  PRIVATE
   XrdUtils )
 
-set_target_properties(
-  ${LIB_XRD_SEC_UNIX}
-  PROPERTIES
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
-
 #-------------------------------------------------------------------------------
 # Install
 #-------------------------------------------------------------------------------
@@ -170,10 +144,4 @@ install(
   xrdsssadmin xrdpwdadmin
   RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
   LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} )
-
-install(
-  FILES
-  ${PROJECT_SOURCE_DIR}/docs/man/xrdsssadmin.8
-  ${PROJECT_SOURCE_DIR}/docs/man/xrdpwdadmin.8
-  DESTINATION ${CMAKE_INSTALL_MANDIR}/man8 )
 endif()
diff --git a/src/XrdSec/XrdSecClient.cc b/src/XrdSec/XrdSecClient.cc
index 12337306482..82bfc61ed7c 100644
--- a/src/XrdSec/XrdSecClient.cc
+++ b/src/XrdSec/XrdSecClient.cc
@@ -50,7 +50,7 @@
 /*                 M i s c e l l a n e o u s   D e f i n e s                  */
 /******************************************************************************/
 
-#define DEBUG(x) {if (DebugON) cerr <<"sec_Client: " <setErrInfo(ENOPROTOOPT, noperr);
-         else cerr <.        */
+/*                                                                            */
+/* The copyright holder's institutional names and contributor's names may not */
+/* be used to endorse or promote products derived from this software without  */
+/* specific prior written permission of the institution or contributor.       */
+/******************************************************************************/
+
+class XrdSecMonitor
+{
+public:
+
+enum WhatInfo {TokenInfo = 0};
+
+//------------------------------------------------------------------------------
+//! Include extra information in the monitoring stream to be associated with
+//! the current mapped user. This object is pointed to via the XrdSecEntity
+//! secMon member.
+//! 
+//! @param  infoT - the enum describing what information is being reported
+//! @param  info  - a null terminate string with the information in cgi format
+//! 
+//! @return true  - Information reported.
+//! @return false - Invalid infoT code or not enabled, call has been ignored.
+//------------------------------------------------------------------------------
+
+virtual bool      Report(WhatInfo infoT, const char *info) = 0;
+
+                  XrdSecMonitor() {}
+virtual          ~XrdSecMonitor() {}
+};
+#endif
diff --git a/src/XrdSec/XrdSecPManager.cc b/src/XrdSec/XrdSecPManager.cc
index 39e30b0895b..96f496b8f49 100644
--- a/src/XrdSec/XrdSecPManager.cc
+++ b/src/XrdSec/XrdSecPManager.cc
@@ -54,7 +54,7 @@
 /*                 M i s c e l l a n e o u s   D e f i n e s                  */
 /******************************************************************************/
 
-#define DEBUG(x) {if (DebugON) cerr <<"sec_PM: " <getErrInfo() != ENOENT) cerr <getErrText() <getErrInfo() != ENOENT) std::cerr <getErrText() <setErrInfo(rc, tlist, n);
-      else {for (i = 0; i < n; i++) cerr <Beg(epname,tident); cerr <End();}
+           {SecTrace->Beg(epname,tident); std::cerr <End();}
 
 #define DEBUG(y) if (QTRACE(Debug)) \
-                    {SecTrace->Beg(epname); cerr <End();}
+                    {SecTrace->Beg(epname); std::cerr <End();}
 #define EPNAME(x) static const char *epname = x;
 
 #else
diff --git a/src/XrdSec/XrdSectestClient.cc b/src/XrdSec/XrdSectestClient.cc
index 7748ffd061f..fc6d0b83e19 100644
--- a/src/XrdSec/XrdSectestClient.cc
+++ b/src/XrdSec/XrdSectestClient.cc
@@ -107,14 +107,14 @@ void help(int);
 /*Make sure no more parameters exist.
 */
    if (optind < argc) 
-      {cerr <<"testClient: Extraneous parameter, '" <addrInfo = &theAddr;
    cred = pp->getCredentials();
    if (!cred)
-      {cerr << "Unable to get credentials," <buffer, cred->size, 1, stdout) != (size_t) cred->size)
-          {cerr << "Unable to write credentials" <getParms(i, opts.host);
-   if (!sect) cerr <<"testServer: No security token for " <Authenticate(&cred, &parmp, &einfo) < 0)
       {rc = einfo.getErrInfo();
-       cerr << "testServer: Authenticate error " <Entity.name ? pp->Entity.name : "?")
+      std::cout <<(pp->Entity.name ? pp->Entity.name : "?")
            <<"@" <<(pp->Entity.host ? pp->Entity.host : "?")
-           <<" prot=" <Entity.prot <Beg(epname); cerr <End();}}
-#define POPTS(t,y)      {if (t) {cerr <<"Secgsi" <Beg(epname); std::cerr <End();}}
+#define POPTS(t,y)      {if (t) {std::cerr <<"Secgsi" <Cleanup(1);
+   if (proxyChain) proxyChain->Cleanup();
    SafeDelete(proxyChain);    // Chain with delegated proxies
    SafeFree(expectedHost);
 
@@ -2819,13 +2819,13 @@ XrdSecProtocol *XrdSecProtocolgsiObject(const char              mode,
       if (erp) 
          erp->setErrInfo(ENOMEM, msg);
       else 
-         cerr <Options & kOptsFwdPxy)) {
       // Create a new proxy chain
       hs->PxyChain = new X509Chain();
+      // The new chain must be deleted if still in the handshake info
+      // when the info is destroyed
+      hs->Options |= kOptsDelPxy;
       // Add the current proxy
       if ((*ParseBucket)(bck, hs->PxyChain) > 1) {
          // Reorder it
@@ -3912,21 +3915,34 @@ int XrdSecProtocolgsi::ServerDoCert(XrdSutBuffer *br,  XrdSutBuffer **bm,
             XrdCryptoRSA *krPXp = 0;
             if ((*X509CreateProxyReq)(hs->PxyChain->End(), &rPXp, &krPXp) == 0) {
                // Save key in the cache
-               hs->Cref->buf4.buf = (char *)krPXp;
+               hs->Cref->buf4.len = krPXp->GetPrilen() + 1;
+               hs->Cref->buf4.buf = new char[hs->Cref->buf4.len];
+               if (krPXp->ExportPrivate(hs->Cref->buf4.buf, hs->Cref->buf4.len) != 0) {
+                  delete krPXp;
+                  delete rPXp;
+                  if (hs->PxyChain) hs->PxyChain->Cleanup();
+                  SafeDelete(hs->PxyChain);
+                  cmsg = "cannot export private key of the proxy request!";
+                  return -1;
+               }
                // Prepare export bucket for request
                XrdSutBucket *bckr = rPXp->Export();
                // Add it to the main list
                if ((*bm)->AddBucket(bckr) != 0) {
+                  if (hs->PxyChain) hs->PxyChain->Cleanup();
                   SafeDelete(hs->PxyChain);
                   NOTIFY("WARNING: proxy req: problem adding bucket to main buffer");
                }
+               delete krPXp;
                delete rPXp;
             } else {
+               if (hs->PxyChain) hs->PxyChain->Cleanup();
                SafeDelete(hs->PxyChain);
                NOTIFY("WARNING: proxy req: problem creating request");
             }
          }
       } else {
+         if (hs->PxyChain) hs->PxyChain->Cleanup();
          SafeDelete(hs->PxyChain);
          NOTIFY("WARNING: proxy req: wrong number of certificates");
       }
@@ -4037,8 +4053,12 @@ int XrdSecProtocolgsi::ServerDoSigpxy(XrdSutBuffer *br,  XrdSutBuffer **bm,
          return 0;
       }
       // Set full PKI
-      XrdCryptoRSA *knpx = (XrdCryptoRSA *)(hs->Cref->buf4.buf);
-      npx->SetPKI((XrdCryptoX509data)(knpx->Opaque()));
+      XrdCryptoRSA *const knpx = npx->PKI();
+      if (!knpx || knpx->ImportPrivate(hs->Cref->buf4.buf, hs->Cref->buf4.len) != 0) {
+        delete npx;
+        cmsg = "could not import private key into signed request";
+        return 0;
+      }
       // Add the new proxy ecert to the chain
       pxyc->PushBack(npx);
    }
diff --git a/src/XrdSecgsi/XrdSecProtocolgsi.hh b/src/XrdSecgsi/XrdSecProtocolgsi.hh
index 5c5ccfa0cd6..2c79f5aebe8 100644
--- a/src/XrdSecgsi/XrdSecProtocolgsi.hh
+++ b/src/XrdSecgsi/XrdSecProtocolgsi.hh
@@ -112,7 +112,8 @@ enum kgsiHandshakeOpts {
    kOptsPxFile     = 16,     // 0x0010: Save delegated proxies in file
    kOptsDelChn     = 32,     // 0x0020: Delete chain
    kOptsPxCred     = 64,     // 0x0040: Save delegated proxies as credentials
-   kOptsCreatePxy  = 128     // 0x0080: Request a client proxy
+   kOptsCreatePxy  = 128,    // 0x0080: Request a client proxy
+   kOptsDelPxy     = 256     // 0x0100: Delete the proxy PxyChain
 };
 
 // Error codes
@@ -540,9 +541,14 @@ public:
                      XrdSecProtocolgsi::stackCRL->Del(Crl);
                      Crl = 0;
                   }
-                  // The proxy chain is owned by the proxy cache; invalid proxies are
-                  // detected (and eventually removed) by QueryProxy
-                  PxyChain = 0;
+                  if (Options & kOptsDelPxy) {
+                     if (PxyChain) PxyChain->Cleanup();
+                     SafeDelete(PxyChain);
+                  } else {
+                     // The proxy chain is owned by the proxy cache; invalid proxies
+                     // are detected (and eventually removed) by QueryProxy
+                     PxyChain = 0;
+                  }
                   SafeDelete(Parms); }
    void Dump(XrdSecProtocolgsi *p = 0);
 };
diff --git a/src/XrdSecgsi/XrdSecgsiGMAPFunDN.cc b/src/XrdSecgsi/XrdSecgsiGMAPFunDN.cc
index 38835f0e316..99bd139093d 100644
--- a/src/XrdSecgsi/XrdSecgsiGMAPFunDN.cc
+++ b/src/XrdSecgsi/XrdSecgsiGMAPFunDN.cc
@@ -51,7 +51,7 @@ static XrdOucTrace  *dnTrace = 0;
 
 #define TRACE_Authen   0x0002
 #define EPNAME(x)    static const char *epname = x;
-#define PRINT(y)    {if (dnTrace) {dnTrace->Beg(epname); cerr <End();}}
+#define PRINT(y)    {if (dnTrace) {dnTrace->Beg(epname); std::cerr <End();}}
 #define DEBUG(y)   if (dnTrace && (dnTrace->What & TRACE_Authen)) PRINT(y)
 
 
@@ -204,7 +204,7 @@ int XrdSecgsiGMAPInit(const char *parms)
          if (len < 2) continue;
          if (l[0] == '#') continue;
          if (l[len-1] == '\n') l[len-1] = '\0';
-         if (sscanf(l, "%4096s %256s", val, usr) >= 2) {
+         if (sscanf(l, "%4095s %255s", val, usr) >= 2) {
             XrdOucString stype = "matching";
             char *p = &val[0];
             int type = kFull;
diff --git a/src/XrdSecgsi/XrdSecgsiProxy.cc b/src/XrdSecgsi/XrdSecgsiProxy.cc
index 04879af73af..171645fb537 100644
--- a/src/XrdSecgsi/XrdSecgsiProxy.cc
+++ b/src/XrdSecgsi/XrdSecgsiProxy.cc
@@ -61,7 +61,7 @@
 
 #include "XrdSecgsi/XrdSecgsiTrace.hh"
 
-#define PRT(x) {cerr <What & TRACE_ ## act))
 #define PRINT(y)    {if (gsiTrace) {gsiTrace->Beg(epname); \
-                                       cerr <End();}}
+                                       std::cerr <End();}}
 #define TRACE(act,x) if (QTRACE(act)) PRINT(x)
 #define NOTIFY(y)    TRACE(Debug,y)
 #define DEBUG(y)     TRACE(Authen,y)
diff --git a/src/XrdSecgsi/XrdSecgsitest.cc b/src/XrdSecgsi/XrdSecgsitest.cc
index b721079b721..98a3fdda78c 100644
--- a/src/XrdSecgsi/XrdSecgsitest.cc
+++ b/src/XrdSecgsi/XrdSecgsitest.cc
@@ -62,7 +62,7 @@
 //
 // Globals 
 
-// #define PRINT(x) {cerr < 0) {
+      printf("|| %s ", t);
+   } else {
+      printf("||  ");
+   }
    for (; i < np ; i++) { printf("."); }
    printf("  %s\n", (ok ? "PASSED" : "FAILED"));
 }
@@ -256,6 +260,10 @@ int main( int argc, char **argv )
       exit(1);
    }
 
+   // use recreated proxy certificate if it a proxy was not already set
+   if (!xPX)
+     xPX = xPXp;
+
    //
    pline("");
    pline("Load CA certificates");
@@ -276,6 +284,8 @@ int main( int argc, char **argv )
          pdots("Loading CA certificate", 1);
       } else {
          pdots("Loading CA certificate", 0);
+         rCAfound = 0;
+         break;
       }
       // Check if self-signed
       if (!strcmp(xCA[nCA]->IssuerHash(), xCA[nCA]->SubjectHash())) {
@@ -325,7 +335,7 @@ int main( int argc, char **argv )
    pline("Testing ExportChain");
    XrdCryptoX509ExportChain_t ExportChain = gCryptoFactory->X509ExportChain();
    XrdSutBucket *chainbck = 0;
-   if (ExportChain) {
+   if (ExportChain && chain->End()) {
       chainbck = (*ExportChain)(chain, 0);
       pdots("Attach to X509ExportChain", 1);
    } else {
diff --git a/src/XrdSeckrb5.cmake b/src/XrdSeckrb5.cmake
index 30dceff0e8e..0597c8a6077 100644
--- a/src/XrdSeckrb5.cmake
+++ b/src/XrdSeckrb5.cmake
@@ -1,6 +1,3 @@
-include_directories( ${KERBEROS5_INCLUDE_DIR} )
-include( XRootDCommon )
-
 #-------------------------------------------------------------------------------
 # The XrdSeckrb5 module
 #-------------------------------------------------------------------------------
@@ -13,14 +10,11 @@ add_library(
 
 target_link_libraries(
   ${LIB_XRD_SEC_KRB5}
+  PRIVATE
   XrdUtils
   ${KERBEROS5_LIBRARIES} )
 
-set_target_properties(
-  ${LIB_XRD_SEC_KRB5}
-  PROPERTIES
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
+target_include_directories( ${LIB_XRD_SEC_KRB5} PRIVATE ${KERBEROS5_INCLUDE_DIR} )
 
 #-------------------------------------------------------------------------------
 # Install
diff --git a/src/XrdSeckrb5/XrdSecProtocolkrb5.cc b/src/XrdSeckrb5/XrdSecProtocolkrb5.cc
index 0bca2731eba..9ffcdcb0f0a 100644
--- a/src/XrdSeckrb5/XrdSecProtocolkrb5.cc
+++ b/src/XrdSeckrb5/XrdSecProtocolkrb5.cc
@@ -78,8 +78,8 @@ extern "C" {
 
 #define XrdSecMAXPATHLEN      4096
 
-#define CLDBG(x) if (client_options & XrdSecDEBUG) cerr <<"Seckrb5: " <setErrInfo(rc, msgv, i);
-      else {for (k = 0; k < i; k++) cerr <setErrInfo(EINVAL, msg);
-               else cerr <setErrInfo(EINVAL, msg);
-          else cerr <setErrInfo(EINVAL, msg);
-              else cerr <setErrInfo(ENOMEM, msg);
-          else cerr <Beg(epname); cerr <End();}}
+#define POPTS(t,y)    {if (t) {t->Beg(epname); std::cerr <End();}}
 #else
 #define POPTS(t,y)
 #endif
@@ -1916,13 +1916,13 @@ XrdSecProtocol *XrdSecProtocolpwdObject(const char              mode,
       if (erp) 
          erp->setErrInfo(ENOMEM, msg);
       else 
-         cerr <Beg(epname); cerr <End();}}
+#define PRINT(y) {{SecTrace->Beg(epname); std::cerr <End();}}
 #else
 #define PRINT(y) { }
 #endif
diff --git a/src/XrdSecpwd/XrdSecpwdSrvAdmin.cc b/src/XrdSecpwd/XrdSecpwdSrvAdmin.cc
index fe839aaa3fa..b0f7751b7a6 100644
--- a/src/XrdSecpwd/XrdSecpwdSrvAdmin.cc
+++ b/src/XrdSecpwd/XrdSecpwdSrvAdmin.cc
@@ -212,7 +212,7 @@ bool GetEntry(XrdSutPFile *ff, XrdOucString tag,
 bool AskConfirm(const char *msg1, bool defact, const char *msg2 = 0);
 int LocateFactoryIndex(char *tag, int &id);
 
-#define PRT(x) {cerr <What & TRACE_ ## act))
 #define PRINT(y)    {if (pwdTrace) {pwdTrace->Beg(epname); \
-                                       cerr <End();}}
+                                       std::cerr <End();}}
 #define TRACE(act,x) if (QTRACE(act)) PRINT(x)
 #define NOTIFY(y)    TRACE(Debug,y)
 #define DEBUG(y)     TRACE(Authen,y)
diff --git a/src/XrdSecsss/XrdSecsssEnt.cc b/src/XrdSecsss/XrdSecsssEnt.cc
index fdfa8657bc4..cfad10aceba 100644
--- a/src/XrdSecsss/XrdSecsssEnt.cc
+++ b/src/XrdSecsss/XrdSecsssEnt.cc
@@ -235,7 +235,7 @@ bool XrdSecsssEnt::Serialize()
 // must be at the end because it can optionally be pruned when returned.
 //
    if (eP->credslen && eP->credslen <= XrdSecsssRR_Data::MaxCSz)
-      {tLen += eP->credslen;
+      {tLen += eP->credslen + 3;
        incCreds = true;
       }
 
diff --git a/src/XrdSecunix/XrdSecProtocolunix.cc b/src/XrdSecunix/XrdSecProtocolunix.cc
index d9a1c8bea30..6abf1018bc6 100644
--- a/src/XrdSecunix/XrdSecProtocolunix.cc
+++ b/src/XrdSecunix/XrdSecProtocolunix.cc
@@ -147,7 +147,7 @@ int XrdSecProtocolunix::Authenticate(XrdSecCredentials *cred,
                 "Secunix: Authentication protocol id mismatch (unix != %.4s).",
                 cred->buffer);
        if (erp) erp->setErrInfo(EINVAL, msg);
-          else cerr <setErrInfo(ENOMEM, msg);
-          else cerr <
-#include 
+
 #include 
 #include 
 #include 
@@ -40,13 +39,19 @@
 #include 
 #include 
 #include 
-#include 
 #include 
 #include 
+
+#ifndef __FreeBSD__
+#include 
+#endif
+
 #include 
 #include 
 #include 
 #include 
+#include 
+#include 
 
 #include "XrdVersion.hh"
 
@@ -324,8 +329,7 @@ XrdSecCredentials *XrdSecProtocolztn::findToken(XrdOucErrInfo *erp,
 
         if (Vec[i].beginswith('/') == 1)
            {char tokPath[MAXPATHLEN+8];
-            snprintf(tokPath, sizeof(tokPath), tokName,
-                     Vec[i].length(), int(geteuid()));
+            snprintf(tokPath, sizeof(tokPath), tokName, int(geteuid()));
             resp = readToken(erp, tokPath, isbad);
             if (resp || isbad) return resp;
             continue;
diff --git a/src/XrdSecztn/XrdSecztn.cc b/src/XrdSecztn/XrdSecztn.cc
index dcb955e1f58..036baed646e 100644
--- a/src/XrdSecztn/XrdSecztn.cc
+++ b/src/XrdSecztn/XrdSecztn.cc
@@ -25,10 +25,14 @@
 // heavily edited to solve this particular problem. For more info see:
 // https://github.com/pokowaka/jwt-cpp
 
-#include 
 #include 
+#include 
 #include 
 
+#ifndef __FreeBSD__
+#include 
+#endif
+
 #define WHITESPACE 64
 #define EQUALS 65
 #define INVALID 66
diff --git a/src/XrdServer.cmake b/src/XrdServer.cmake
index d5d2e453347..b826eb2b43f 100644
--- a/src/XrdServer.cmake
+++ b/src/XrdServer.cmake
@@ -1,5 +1,4 @@
 
-include( XRootDCommon )
 
 #-------------------------------------------------------------------------------
 # Plugin version (this protocol loaded eithr as a plugin or as builtin).
@@ -188,6 +187,7 @@ add_library(
 
 target_link_libraries(
   XrdServer
+  PRIVATE
   XrdUtils
   ${CMAKE_DL_LIBS}
   ${CMAKE_THREAD_LIBS_INIT}
@@ -199,9 +199,7 @@ set_target_properties(
   XrdServer
   PROPERTIES
   VERSION   ${XRD_SERVER_VERSION}
-  SOVERSION ${XRD_SERVER_SOVERSION}
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
+  SOVERSION ${XRD_SERVER_SOVERSION} )
 
 #-------------------------------------------------------------------------------
 # The XRootD protocol plugin
@@ -213,16 +211,11 @@ add_library(
 
 target_link_libraries(
   ${LIB_XRD_PROTOCOL}
+  PRIVATE
   XrdServer
   XrdUtils
   ${EXTRA_LIBS} )
 
-set_target_properties(
-  ${LIB_XRD_PROTOCOL}
-  PROPERTIES
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
-
 #-------------------------------------------------------------------------------
 # Install
 #-------------------------------------------------------------------------------
diff --git a/src/XrdSfs/XrdSfsGPFile.hh b/src/XrdSfs/XrdSfsGPFile.hh
index 6241d21a481..e38d267c9ec 100644
--- a/src/XrdSfs/XrdSfsGPFile.hh
+++ b/src/XrdSfs/XrdSfsGPFile.hh
@@ -29,6 +29,8 @@
 /* specific prior written permission of the institution or contributor.       */
 /******************************************************************************/
 
+#include 
+
 class XrdSfsGPInfo;
 
 class XrdSfsGPFile
diff --git a/src/XrdSfs/XrdSfsInterface.hh b/src/XrdSfs/XrdSfsInterface.hh
index 31340182b6e..c0ec1cb9a5d 100644
--- a/src/XrdSfs/XrdSfsInterface.hh
+++ b/src/XrdSfs/XrdSfsInterface.hh
@@ -99,6 +99,7 @@
 #define SFS_FSCTL_STATCC  5 // Return Cluster Config status
 #define SFS_FSCTL_PLUGIN  8 // Return Implementation Dependent Data
 #define SFS_FSCTL_PLUGIO 16 // Return Implementation Dependent Data
+#define SFS_FSCTL_PLUGXC 32 // Perform cache oriented operation
 
 // Return values for integer & XrdSfsXferSize returning XrdSfs methods
 //
@@ -156,12 +157,15 @@ enum XrdSfsFileExistence
 
 class XrdOucTList;
 
-struct XrdSfsFSctl //!< SFS_FSCTL_PLUGIN/PLUGIO parms
+struct XrdSfsFSctl //!< SFS_FSCTL_PLUGIN/PLUGIO/PLUGXC parms
 {
- const char            *Arg1;      //!< PLUGIO, PLUGIN
+ const char            *Arg1;      //!< PLUGINO, PLUGION, PLUGXC
        int              Arg1Len;   //!< Length
-       int              Arg2Len;   //!< Length
+       int              Arg2Len;   //!< Length  or -count of args in extension
+ union{
  const char            *Arg2;      //!< PLUGIN  opaque string
+ const char           **ArgP;      //!< PLUGXC  argument list extension
+      };
 };
 
 struct XrdSfsPrep  //!< Prepare parameters
@@ -1253,7 +1257,7 @@ virtual int            stat(const char               *Name,
 //!
 //! @return One of SFS_OK, SFS_ERROR, SFS_REDIRECT, SFS_STALL, or SFS_STARTED
 //!         When SFS_OK is returned, mode must contain mode information. If
-//!         teh mode is -1 then it is taken as an offline file.
+//!         the mode is -1 then it is taken as an offline file.
 //-----------------------------------------------------------------------------
 
 virtual int            stat(const char               *path,
diff --git a/src/XrdSsi.cmake b/src/XrdSsi.cmake
index da7f36c94fc..a5f00d92994 100644
--- a/src/XrdSsi.cmake
+++ b/src/XrdSsi.cmake
@@ -1,5 +1,4 @@
 
-include( XRootDCommon )
 
 #-------------------------------------------------------------------------------
 # Modules
@@ -50,6 +49,7 @@ XrdSsi/XrdSsiUtils.cc                  XrdSsi/XrdSsiUtils.hh)
 
 target_link_libraries(
   XrdSsiLib
+  PRIVATE
   XrdCl
   XrdUtils
   ${CMAKE_THREAD_LIBS_INIT} )
@@ -58,8 +58,7 @@ set_target_properties(
   XrdSsiLib
   PROPERTIES
   VERSION   ${XRD_SSI_LIB_VERSION}
-  SOVERSION ${XRD_SSI_LIB_SOVERSION}
-  LINK_INTERFACE_LIBRARIES "" )
+  SOVERSION ${XRD_SSI_LIB_SOVERSION} )
 
 #-------------------------------------------------------------------------------
 # The XrdSsiShMap library
@@ -73,6 +72,7 @@ XrdSsi/XrdSsiShMat.cc                  XrdSsi/XrdSsiShMat.hh)
 
 target_link_libraries(
   XrdSsiShMap
+  PRIVATE
   XrdUtils
   ZLIB::ZLIB
   ${CMAKE_THREAD_LIBS_INIT} )
@@ -81,8 +81,7 @@ set_target_properties(
   XrdSsiShMap
   PROPERTIES
   VERSION   ${XRD_SSI_SHMAP_VERSION}
-  SOVERSION ${XRD_SSI_SHMAP_SOVERSION}
-  LINK_INTERFACE_LIBRARIES "" )
+  SOVERSION ${XRD_SSI_SHMAP_SOVERSION} )
 
 #-------------------------------------------------------------------------------
 # The XrdSsi plugin
@@ -101,16 +100,11 @@ add_library(
 
 target_link_libraries(
   ${LIB_XRD_SSI}
+  PRIVATE
   XrdSsiLib
   XrdUtils
   XrdServer )
 
-set_target_properties(
-  ${LIB_XRD_SSI}
-  PROPERTIES
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
-
 #-------------------------------------------------------------------------------
 # The XrdSsiLog plugin
 #-------------------------------------------------------------------------------
@@ -122,16 +116,11 @@ add_library(
 
 target_link_libraries(
   ${LIB_XRD_SSILOG}
+  PRIVATE
   XrdSsiLib
   XrdUtils
   XrdServer )
 
-set_target_properties(
-  ${LIB_XRD_SSILOG}
-  PROPERTIES
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
-
 #-------------------------------------------------------------------------------
 # Install
 #-------------------------------------------------------------------------------
diff --git a/src/XrdSsi/XrdSsiClient.cc b/src/XrdSsi/XrdSsiClient.cc
index c386206e43e..049443ab493 100644
--- a/src/XrdSsi/XrdSsiClient.cc
+++ b/src/XrdSsi/XrdSsiClient.cc
@@ -76,7 +76,7 @@ extern XrdSsiLogger::MCB_t *msgCBCl;
        Atomic(int)   contactN(1);
        short         maxTCB   = 300;
        short         maxCLW   =  30;
-       short         maxPEL   =   3;
+       short         maxPEL   =  10;
        Atomic(bool)  initDone(false);
        bool          dsTTLSet = false;
        bool          reqTOSet = false;
diff --git a/src/XrdSsi/XrdSsiCms.cc b/src/XrdSsi/XrdSsiCms.cc
index 694708033dd..b2966280c94 100644
--- a/src/XrdSsi/XrdSsiCms.cc
+++ b/src/XrdSsi/XrdSsiCms.cc
@@ -61,7 +61,7 @@ XrdSsiCms::XrdSsiCms(XrdCmsClient *cmsP) : theCms(cmsP)
    tP     = stP;
    while(tP) {manNum++; tP = tP->next;}
 
-// Allocate an array of teh right size
+// Allocate an array of the right size
 //
    manList = new char*[manNum];
 
diff --git a/src/XrdSsi/XrdSsiLogger.cc b/src/XrdSsi/XrdSsiLogger.cc
index 01cfcda5beb..696ab96c63c 100644
--- a/src/XrdSsi/XrdSsiLogger.cc
+++ b/src/XrdSsi/XrdSsiLogger.cc
@@ -215,6 +215,6 @@ const char *XrdSsiLogger::TBeg() {return Logger->traceBeg();}
   
 void XrdSsiLogger::TEnd()
 {
-   cerr <traceEnd();
 }
diff --git a/src/XrdSsi/XrdSsiLogger.hh b/src/XrdSsi/XrdSsiLogger.hh
index 93032116188..1b3774a7500 100644
--- a/src/XrdSsi/XrdSsiLogger.hh
+++ b/src/XrdSsi/XrdSsiLogger.hh
@@ -118,14 +118,14 @@ enum mcbType {mcbAll=0, mcbClient, mcbServer};
 static bool SetMCB(MCB_t &mcbP, mcbType mcbt=mcbAll);
 
 //-----------------------------------------------------------------------------
-//! Define helper functions to allow ostream cerr output to appear in the log.
+//! Define helper functions to allow std::ostream std::cerr output to appear in the log.
 //! The following two functions are used with the macros below.
 //! The SSI_LOG macro preceedes the message with a time stamp; SSI_SAY does not.
-//! The endl ostream output item is automatically added to all output!
+//! The std::endl std::ostream output item is automatically added to all output!
 //-----------------------------------------------------------------------------
 
-#define SSI_LOG(x) {cerr <getPlugin("XrdSsiLoggerMCB"));
-       if (!msgCB && !theCB) cerr <<"Config " <Persist();
                }
@@ -153,8 +153,8 @@ extern "C"
 XrdSysLogPI_t  XrdSysLogPInit(const char *cfgfn, char **argv, int argc)
           {if (cfgfn && *cfgfn) ConfigLog(cfgfn);
            if (!msgCB)
-              cerr <<"Config '-l@' requires a logmsg callback function "
-                   <<"but it was found!" <size > 0 && bp->buffer) {
             if (pripre) {
                XrdOucString premsg(prepose);
-               cerr << premsg << endl;
+               std::cerr << premsg << std::endl;
                pripre = 0;
             }
             XrdOucString msg(bp->buffer,bp->size);
-            cerr << msg << endl;
+            std::cerr << msg << std::endl;
          }
       }
       // Get next
diff --git a/src/XrdSut/XrdSutTrace.hh b/src/XrdSut/XrdSutTrace.hh
index 388823a2a18..04e863560fc 100644
--- a/src/XrdSut/XrdSutTrace.hh
+++ b/src/XrdSut/XrdSutTrace.hh
@@ -41,7 +41,7 @@
 
 #define QTRACE(act) (sutTrace && (sutTrace->What & sutTRACE_ ## act))
 #define PRINT(y)    {if (sutTrace) {sutTrace->Beg(epname); \
-                                    cerr <End();}}
+                                    std::cerr <End();}}
 #define TRACE(act,x) if (QTRACE(act)) PRINT(x)
 #define DEBUG(y)     TRACE(Debug,y)
 #define EPNAME(x)    static const char *epname = x;
diff --git a/src/XrdSys/XrdSysError.cc b/src/XrdSys/XrdSysError.cc
index 4bfee80eceb..3812cad77a4 100644
--- a/src/XrdSys/XrdSysError.cc
+++ b/src/XrdSys/XrdSysError.cc
@@ -160,14 +160,14 @@ void XrdSysError::Say(const char *txt1, const char *txt2, const char *txt3,
   
 void XrdSysError::TBeg(const char *txt1, const char *txt2, const char *txt3)
 {
- cerr <traceBeg();
- if (txt1) cerr <traceBeg();
+ if (txt1) std::cerr <traceEnd();}
+void XrdSysError::TEnd() {std::cerr <traceEnd();}
diff --git a/src/XrdSys/XrdSysError.hh b/src/XrdSys/XrdSysError.hh
index 0d970db522a..0a08d45f792 100644
--- a/src/XrdSys/XrdSysError.hh
+++ b/src/XrdSys/XrdSysError.hh
@@ -163,7 +163,7 @@ inline const char *SetPrefix(const char *prefix)
                          return oldpfx;
                         }
 
-// TBeg() is used to start a trace on ostream cerr. The TEnd() ends the trace.
+// TBeg() is used to start a trace on std::ostream std::cerr. The TEnd() ends the trace.
 //
 void TBeg(const char *txt1=0, const char *txt2=0, const char *txt3=0);
 void TEnd();
diff --git a/src/XrdSys/XrdSysFAttr.cc b/src/XrdSys/XrdSysFAttr.cc
index 616d0202b39..de4739e5804 100644
--- a/src/XrdSys/XrdSysFAttr.cc
+++ b/src/XrdSys/XrdSysFAttr.cc
@@ -120,7 +120,7 @@ void XrdSysFAttr::Free(XrdSysFAttr::AList *aLP)
 {
    AList *aNP;
 
-// Free all teh structs using free as they were allocated using malloc()
+// Free all the structs using free as they were allocated using malloc()
 //
    while(aLP) {aNP = aLP->Next; free(aLP); aLP = aNP;}
 }
diff --git a/src/XrdSys/XrdSysIOEvents.cc b/src/XrdSys/XrdSysIOEvents.cc
index 8d4b08a83c1..e47445a2321 100644
--- a/src/XrdSys/XrdSysIOEvents.cc
+++ b/src/XrdSys/XrdSysIOEvents.cc
@@ -85,7 +85,7 @@ namespace
 
 #define DO_TRACE(x,fd,y) \
                 {PollerInit::traceMTX.Lock(); \
-                 cerr <<"IOE fd "<chFD,"chan="<chFD,"chan="<< std::hex<<(void*)cP<< std::dec
             <<" inTOQ="<inTOQ)<<" status="<chFD,"chan="<chFD,"chan="<< std::hex<<(void*)cP<< std::dec
             <<" inTOQ="<inTOQ)<<" status="<
 #include 
 
 #ifdef __linux__
@@ -46,6 +47,7 @@
 #include 
 #include 
 #include 
+#include 
 #define fdatasync(x) fsync(x)
 #define MAXNAMELEN NAME_MAX
 #ifndef dirent64
@@ -65,12 +67,18 @@
 #if defined(__FreeBSD__) || (defined(__FreeBSD_kernel__) && defined(__GLIBC__))
 #include 
 #include 
+#if defined(__FreeBSD__)
+#include 
+#else
+#include 
+#endif
 #define MAXNAMELEN NAME_MAX
 #endif
 
 #ifdef __GNU__
 #include 
 #include 
+#include 
 // These are undefined on purpose in GNU/Hurd.
 // The values below are the ones used in Linux.
 // The proper fix is to rewrite the code to not use hardcoded values,
@@ -153,6 +161,22 @@ typedef off_t offset_t;
 typedef off_t off64_t;
 #endif
 
+#if defined(__APPLE__)
+#define bswap_16 OSSwapInt16
+#define bswap_32 OSSwapInt32
+#define bswap_64 OSSwapInt64
+#endif
+
+#if defined(__FreeBSD__)
+#define bswap_16 bswap16
+#define bswap_32 bswap32
+#define bswap_64 bswap64
+#endif
+
+static inline uint16_t bswap(uint16_t x) { return bswap_16(x); }
+static inline uint32_t bswap(uint32_t x) { return bswap_32(x); }
+static inline uint64_t bswap(uint64_t x) { return bswap_64(x); }
+
 // Only sparc platforms have structure alignment problems w/ optimization
 // so the h2xxx() variants are used when converting network streams.
 
diff --git a/src/XrdSys/XrdSysPlugin.cc b/src/XrdSys/XrdSysPlugin.cc
index 5c9abfd0752..46b1d17707b 100644
--- a/src/XrdSys/XrdSysPlugin.cc
+++ b/src/XrdSys/XrdSysPlugin.cc
@@ -496,7 +496,7 @@ bool XrdSysPlugin::VerCmp(XrdVersionInfo &vInfo1,
       else mTxt = " which is incompatible!";
 
    if (!noMsg)
-      cerr <<"Plugin: " <ID); cerr <ID); std::cerr <
 
 #include "XrdOuc/XrdOucUtils.hh"
-#include "XrdSys/XrdSysAtomics.hh"
+#include "XrdSys/XrdSysRAtomic.hh"
 #include "XrdSys/XrdSysError.hh"
 #include "XrdSys/XrdSysPthread.hh"
 #include "XrdSys/XrdSysTimer.hh"
@@ -34,10 +34,6 @@
 #include "XrdTls/XrdTlsContext.hh"
 #include "XrdTls/XrdTlsTrace.hh"
 
-#if __cplusplus >= 201103L
-#include 
-#endif
-
 /******************************************************************************/
 /*                               G l o b a l s                                */
 /******************************************************************************/
@@ -317,9 +313,45 @@ namespace
 extern "C" {
 #endif
 
+template
+struct tlsmix;
+
+template<>
+struct tlsmix {
+  static unsigned long mixer(unsigned long x) {
+    // mixer based on splitmix64
+    x ^= x >> 30;
+    x *= 0xbf58476d1ce4e5b9UL;
+    x ^= x >> 27;
+    x *= 0x94d049bb133111ebUL;
+    x ^= x >> 31;
+    return x;
+  }
+};
+
+template<>
+struct tlsmix {
+  static unsigned long mixer(unsigned long x) {
+    // mixer based on murmurhash3
+    x ^= x >> 16;
+    x *= 0x85ebca6bU;
+    x ^= x >> 13;
+    x *= 0xc2b2ae35U;
+    x ^= x >> 16;
+    return x;
+  }
+};
+
 unsigned long sslTLS_id_callback(void)
 {
-   return (unsigned long)XrdSysThread::ID();
+   // base thread-id on the id given by XrdSysThread;
+   // but openssl 1.0 uses thread-id as a key for looking
+   // up per thread crypto ERR structures in a hash-table.
+   // So mix bits so that the table's hash function gives
+   // better distribution.
+
+   unsigned long x = (unsigned long)XrdSysThread::ID();
+   return tlsmix::mixer(x);
 }
 
 XrdSysMutex *MutexVector = 0;
@@ -364,12 +396,9 @@ const char *sslCiphers = "ECDHE-ECDSA-AES128-GCM-SHA256:"
 const char *sslCiphers = "ALL:!LOW:!EXP:!MD5:!MD2";
 #endif
 
-XrdSysMutex            ctxMutex;
-#if __cplusplus >= 201103L
-std::atomic      initDone( false );
-#else
-bool                   initDone = false;
-#endif
+XrdSysMutex            dbgMutex, tlsMutex;
+XrdSys::RAtomic  initDbgDone{ false };
+bool                   initTlsDone{ false };
 
 /******************************************************************************/
 /*                               I n i t T L S                                */
@@ -377,13 +406,13 @@ bool                   initDone = false;
   
 void InitTLS() // This is strictly a one-time call!
 {
-   XrdSysMutexHelper ctxHelper(ctxMutex);
+   XrdSysMutexHelper tlsHelper(tlsMutex);
 
 // Make sure we are not trying to load the ssl library more than once. This can
 // happen when a server and a client instance happen to be both defined.
 //
-   if (initDone) return;
-   initDone = true;
+   if (initTlsDone) return;
+   initTlsDone = true;
 
 // SSL library initialisation
 //
@@ -538,8 +567,11 @@ int VerCB(int aOK, X509_STORE_CTX *x509P)
 /*                           C o n s t r u c t o r                            */
 /******************************************************************************/
 
-#define FATAL(msg) {Fatal(eMsg, msg); return;}
-#define FATAL_SSL(msg) {Fatal(eMsg, msg, true); return;}
+#define KILL_CTX(x) if (x) {SSL_CTX_free(x); x = 0;}
+
+#define FATAL(msg) {Fatal(eMsg, msg); KILL_CTX(pImpl->ctx); return;}
+
+#define FATAL_SSL(msg) {Fatal(eMsg, msg, true); KILL_CTX(pImpl->ctx); return;}
   
 XrdTlsContext::XrdTlsContext(const char *cert,  const char *key,
                              const char *caDir, const char *caFile,
@@ -572,24 +604,21 @@ XrdTlsContext::XrdTlsContext(const char *cert,  const char *key,
 // Verify that initialzation has occurred. This is not heavy weight as
 // there will usually be no more than two instances of this object.
 //
-   AtomicBeg(ctxMutex);
-#if __cplusplus >= 201103L
-   bool done = initDone.load();
-#else
-   bool done = AtomicGet(initDone);
-#endif
-   AtomicEnd(ctxMutex);
-   if (!done)
-      {const char *dbg;
-       if (!(opts & servr) && (dbg = getenv("XRDTLS_DEBUG")))
-          {int dbgOpts = 0;
-           if (strstr(dbg, "ctx")) dbgOpts |= XrdTls::dbgCTX;
-           if (strstr(dbg, "sok")) dbgOpts |= XrdTls::dbgSOK;
-           if (strstr(dbg, "sio")) dbgOpts |= XrdTls::dbgSIO;
-           if (!dbgOpts) dbgOpts = XrdTls::dbgALL;
-           XrdTls::SetDebug(dbgOpts|XrdTls::dbgOUT);
+   if (!initDbgDone)
+      {XrdSysMutexHelper dbgHelper(dbgMutex);
+       if (!initDbgDone)
+          {const char *dbg;
+           if (!(opts & servr) && (dbg = getenv("XRDTLS_DEBUG")))
+              {int dbgOpts = 0;
+               if (strstr(dbg, "ctx")) dbgOpts |= XrdTls::dbgCTX;
+               if (strstr(dbg, "sok")) dbgOpts |= XrdTls::dbgSOK;
+               if (strstr(dbg, "sio")) dbgOpts |= XrdTls::dbgSIO;
+               if (!dbgOpts) dbgOpts = XrdTls::dbgALL;
+               XrdTls::SetDebug(dbgOpts|XrdTls::dbgOUT);
+              }
+           if ((emsg = Init())) FATAL(emsg);
+           initDbgDone = true;
           }
-       if ((emsg = Init())) FATAL(emsg);
       }
 
 // If no CA cert information is specified and this is not a server context,
@@ -716,7 +745,7 @@ XrdTlsContext::XrdTlsContext(const char *cert,  const char *key,
 
 // Load certificate
 //
-   if (SSL_CTX_use_certificate_file(pImpl->ctx, cert, SSL_FILETYPE_PEM) != 1)
+   if (SSL_CTX_use_certificate_chain_file(pImpl->ctx, cert) != 1)
       FATAL_SSL("Unable to create TLS context; invalid certificate.");
 
 // Load the private key
@@ -893,7 +922,6 @@ void *XrdTlsContext::Session()
    //We have a new context generated by Refresh, so we must use it.
    XrdTlsContext * ctxnew = pImpl->ctxnew;
 
-#if OPENSSL_VERSION_NUMBER < 0x10101000L
    /*X509_STORE *newX509 = SSL_CTX_get_cert_store(ctxnew->pImpl->ctx);
    SSL_CTX_set1_verify_cert_store(pImpl->ctx, newX509);
    SSL_CTX_set1_chain_cert_store(pImpl->ctx, newX509);*/
@@ -909,10 +937,6 @@ void *XrdTlsContext::Session()
    //we just created, we don't want that to happen. We therefore set it to 0.
    //The SSL_free called on the session will cleanup the context for us.
    ctxnew->pImpl->ctx = 0;
-#else
-   X509_STORE *newX509 = SSL_CTX_get_cert_store(ctxnew->pImpl->ctx);
-   SSL_CTX_set1_cert_store(pImpl->ctx, newX509);
-#endif
 
 // Save the generated context and clear it's presence
 //
@@ -1102,4 +1126,4 @@ bool XrdTlsContext::newHostCertificateDetected() {
         }
     }
     return false;
-}
\ No newline at end of file
+}
diff --git a/src/XrdTls/XrdTlsContext.hh b/src/XrdTls/XrdTlsContext.hh
index 2b510378fb8..e6b61b7b828 100644
--- a/src/XrdTls/XrdTlsContext.hh
+++ b/src/XrdTls/XrdTlsContext.hh
@@ -19,7 +19,7 @@
 //------------------------------------------------------------------------------
 
 #include 
-//#include 
+#include 
   
 //----------------------------------------------------------------------------
 // Forward declarations
diff --git a/src/XrdTls/XrdTlsTempCA.cc b/src/XrdTls/XrdTlsTempCA.cc
index 37431dee9f3..1944ad79819 100644
--- a/src/XrdTls/XrdTlsTempCA.cc
+++ b/src/XrdTls/XrdTlsTempCA.cc
@@ -50,8 +50,8 @@
 #include 
 
 namespace {
-    
-typedef std::unique_ptr file_smart_ptr;
+
+typedef std::unique_ptr file_smart_ptr;
 
 
 static uint64_t monotonic_time_s() {
@@ -60,13 +60,28 @@ static uint64_t monotonic_time_s() {
   return tp.tv_sec + (tp.tv_nsec >= 500000000);
 }
 
+/**
+ * Class managing the CRL or CA output file pointer. It is a RAII-style class that opens the output
+ * file in the constructor and close the file when the instance is destroyed
+ */
+class Set {
+public:
+  Set(int output_fd, XrdSysError & err) : m_log(err),m_output_fp(file_smart_ptr(fdopen(XrdSysFD_Dup(output_fd), "w"), &fclose)){
+    if(!m_output_fp.get()) {
+      m_output_fp.reset();
+    }
+  }
+  virtual ~Set() = default;
+protected:
+  // Reference to the logging that can be used by the inheriting classes.
+  XrdSysError &m_log;
+  // Pointer to the CA or CRL output file
+  file_smart_ptr m_output_fp;
+};
 
-class CASet {
+class CASet : public Set {
 public:
-    CASet(int output_fd, XrdSysError &err)
-    : m_log(err),
-      m_output_fd(output_fd)
-    {}
+    CASet(int output_fd, XrdSysError &err):Set(output_fd,err){}
 
     /**
      * Given an open file descriptor pointing to
@@ -82,13 +97,11 @@ class CASet {
     bool processFile(file_smart_ptr &fd, const std::string &fname);
 
 private:
-    XrdSysError &m_log;
 
         // Grid CA directories tend to keep everything in triplicate;
         // we keep a unique hash of all known CAs so we write out each
         // one only once.
     std::unordered_set m_known_cas;
-    const int m_output_fd;
 };
 
 
@@ -101,9 +114,8 @@ CASet::processFile(file_smart_ptr &fp, const std::string &fname)
     XrdCryptosslX509ParseFile(fp.get(), &chain, fname.c_str());
 
     auto ca = chain.Begin();
-    file_smart_ptr outputfp(fdopen(XrdSysFD_Dup(m_output_fd), "w"), &fclose);
-    if (!outputfp.get()) {
-        m_log.Emsg("CAset", "Failed to reopen file for output", fname.c_str());
+    if (!m_output_fp.get()) {
+        m_log.Emsg("CAset", "No output file has been opened", fname.c_str());
         chain.Cleanup();
         return false;
     }
@@ -121,28 +133,23 @@ CASet::processFile(file_smart_ptr &fp, const std::string &fname)
         //m_log.Emsg("CAset", "New CA with hash", fname.c_str(), hash_ptr);
         m_known_cas.insert(hash_ptr);
 
-        if (XrdCryptosslX509ToFile(ca, outputfp.get(), fname.c_str())) {
+        if (XrdCryptosslX509ToFile(ca, m_output_fp.get(), fname.c_str())) {
             m_log.Emsg("CAset", "Failed to write out CA", fname.c_str());
             chain.Cleanup();
             return false;
         }
         ca = chain.Next();
     }
-    fflush(outputfp.get());
+    fflush(m_output_fp.get());
     chain.Cleanup();
 
     return true;
 }
 
 
-class CRLSet {
+class CRLSet : public Set {
 public:
-    CRLSet(int output_fd, XrdSysError &err)
-    : m_log(err),
-      m_output_fd(output_fd),
-      m_atLeastOneValidCRLFound(false)
-    {}
-
+    CRLSet(int output_fd, XrdSysError &err):Set(output_fd,err){}
     /**
      * Given an open file descriptor pointing to
      * a file potentially containing a CRL, process it
@@ -161,28 +168,34 @@ class CRLSet {
      * processFile(...) method, false otherwise
      */
     bool atLeastOneValidCRLFound() const;
+    /**
+     * https://github.com/xrootd/xrootd/issues/2065
+     * To mitigate that issue, we need to defer the insertion of the CRLs that contain
+     * critical extensions at the end of the bundled CRL file
+     * @return true on success.
+     */
+    bool processCRLWithCriticalExt();
 
 private:
-    XrdSysError &m_log;
 
         // Grid CA directories tend to keep everything in triplicate;
         // we keep a unique hash of all known CRLs so we write out each
         // one only once.
     std::unordered_set m_known_crls;
-    const int m_output_fd;
     std::atomic m_atLeastOneValidCRLFound;
+    //Store the CRLs containing critical extensions to defer their insertion
+    //at the end of the bundled CRL file. Issue https://github.com/xrootd/xrootd/issues/2065
+    std::vector> m_crls_critical_extension;
 };
 
 
 bool
 CRLSet::processFile(file_smart_ptr &fp, const std::string &fname)
 {
-    file_smart_ptr outputfp(fdopen(dup(m_output_fd), "w"), &fclose);
-    if (!outputfp.get()) {
-        m_log.Emsg("CAset", "Failed to reopen file for output", fname.c_str());
+    if (!m_output_fp.get()) {
+        m_log.Emsg("CRLSet", "No output file has been opened", fname.c_str());
         return false;
     }
-
     // Assume we can safely ignore a failure to parse; we load every file in
     // the directory and that will naturally include a number of non-CRL files.
     for (std::unique_ptr xrd_crl(new XrdCryptosslX509Crl(fp.get(), fname.c_str()));
@@ -202,13 +215,20 @@ CRLSet::processFile(file_smart_ptr &fp, const std::string &fname)
         //m_log.Emsg("CRLset", "New CRL with hash", fname.c_str(), hash_ptr);
         m_known_crls.insert(hash_ptr);
 
-        if (!xrd_crl->ToFile(outputfp.get())) {
+        if(xrd_crl->hasCriticalExtension()) {
+          // Issue https://github.com/xrootd/xrootd/issues/2065
+          // This CRL will be put at the end of the bundled file
+          m_crls_critical_extension.emplace_back(std::move(xrd_crl));
+        } else {
+          // No critical extension found on that CRL, just insert it on the CRL bundled file
+          if (!xrd_crl->ToFile(m_output_fp.get())) {
             m_log.Emsg("CRLset", "Failed to write out CRL", fname.c_str());
-            fflush(outputfp.get());
+            fflush(m_output_fp.get());
             return false;
+          }
         }
     }
-    fflush(outputfp.get());
+    fflush(m_output_fp.get());
 
     return true;
 }
@@ -217,6 +237,24 @@ bool CRLSet::atLeastOneValidCRLFound() const {
     return m_atLeastOneValidCRLFound;
 }
 
+bool CRLSet::processCRLWithCriticalExt() {
+  if(!m_crls_critical_extension.empty()) {
+    if (!m_output_fp.get()) {
+      m_log.Emsg("CRLSet", "No output file has been opened to add CRLs with critical extension");
+      return false;
+    }
+    for (const auto &crl: m_crls_critical_extension) {
+      if (!crl->ToFile(m_output_fp.get())) {
+        m_log.Emsg("CRLset", "Failed to write out CRL with critical extension", crl->ParentFile());
+        fflush(m_output_fp.get());
+        return false;
+      }
+    }
+    fflush(m_output_fp.get());
+  }
+  return true;
+}
+
 }
 
 
@@ -362,8 +400,6 @@ XrdTlsTempCA::Maintenance()
         m_log.Emsg("TempCA", "Failed to create a new temp CA / CRL file");
         return false;
     }
-    CASet ca_builder(new_file->getCAFD(), m_log);
-    CRLSet crl_builder(new_file->getCRLFD(), m_log);
 
     int fddir = XrdSysFD_Open(m_ca_dir.c_str(), O_DIRECTORY);
     if (fddir < 0) {
@@ -379,7 +415,10 @@ XrdTlsTempCA::Maintenance()
 
     struct dirent *result;
     errno = 0;
-    while ((result = readdir(dirp))) {
+    {
+      CASet ca_builder(new_file->getCAFD(), m_log);
+      CRLSet crl_builder(new_file->getCRLFD(), m_log);
+      while ((result = readdir(dirp))) {
         //m_log.Emsg("Will parse file for CA certificates", result->d_name);
         if (result->d_name[0] == '.') {continue;}
         if (result->d_type != DT_REG)
@@ -409,23 +448,29 @@ XrdTlsTempCA::Maintenance()
             m_log.Emsg("Maintenance", "Failed to process file for CRLs", result->d_name);
         }
         errno = 0;
-    }
-    if (errno) {
+      }
+      if (errno) {
         m_log.Emsg("Maintenance", "Failure during readdir", strerror(errno));
         closedir(dirp);
         return false;
+      }
+      closedir(dirp);
+
+      if (!crl_builder.processCRLWithCriticalExt()) {
+        m_log.Emsg("Maintenance", "Failed to insert CRLs with critical extension for CRLs", result->d_name);
+      }
+      m_atLeastOneCRLFound = crl_builder.atLeastOneValidCRLFound();
     }
-    closedir(dirp);
 
     if (!new_file->commit()) {
-        m_log.Emsg("Mainteance", "Failed to finalize new CA / CRL files");
+        m_log.Emsg("Maintenance", "Failed to finalize new CA / CRL files");
         return false;
     }
     //m_log.Emsg("Maintenance", "Successfully created CA and CRL files", new_file->getCAFilename().c_str(),
     //    new_file->getCRLFilename().c_str());
     m_ca_file.reset(new std::string(new_file->getCAFilename()));
     m_crl_file.reset(new std::string(new_file->getCRLFilename()));
-    m_atLeastOneCRLFound = crl_builder.atLeastOneValidCRLFound();
+
     return true;
 }
 
diff --git a/src/XrdTpc.cmake b/src/XrdTpc.cmake
index 03e356ea12e..b3e5f46ce48 100644
--- a/src/XrdTpc.cmake
+++ b/src/XrdTpc.cmake
@@ -1,4 +1,3 @@
-include( XRootDCommon )
 
 #-------------------------------------------------------------------------------
 # Modules
@@ -13,8 +12,6 @@ if( BUILD_TPC )
   #-----------------------------------------------------------------------------
   # The XrdHttp library
   #-----------------------------------------------------------------------------
-  include_directories( ${CURL_INCLUDE_DIRS} )
-
   # On newer versions of libcurl, we can use pipelining of requests.
   include (CheckCSourceCompiles)
   SET( CMAKE_REQUIRED_INCLUDES "${CURL_INCLUDE_DIRS}" )
@@ -41,10 +38,12 @@ if( BUILD_TPC )
     XrdTpc/XrdTpcCurlMulti.cc     XrdTpc/XrdTpcCurlMulti.hh
     XrdTpc/XrdTpcState.cc         XrdTpc/XrdTpcState.hh
     XrdTpc/XrdTpcStream.cc        XrdTpc/XrdTpcStream.hh
-    XrdTpc/XrdTpcTPC.cc           XrdTpc/XrdTpcTPC.hh)
+    XrdTpc/XrdTpcTPC.cc           XrdTpc/XrdTpcTPC.hh
+    XrdTpc/XrdTpcPMarkManager.cc  XrdTpc/XrdTpcPMarkManager.hh)
 
   target_link_libraries(
     ${LIB_XRD_TPC}
+    PRIVATE
     XrdServer
     XrdUtils
     XrdHttpUtils
@@ -52,6 +51,8 @@ if( BUILD_TPC )
     ${CMAKE_THREAD_LIBS_INIT}
     ${CURL_LIBRARIES} )
 
+  target_include_directories( ${LIB_XRD_TPC} PRIVATE ${CURL_INCLUDE_DIRS} )
+
   if( MacOSX )
     set( TPC_LINK_FLAGS, "-Wl" )
   else()
@@ -61,8 +62,6 @@ if( BUILD_TPC )
   set_target_properties(
     ${LIB_XRD_TPC}
     PROPERTIES
-    INTERFACE_LINK_LIBRARIES ""
-    LINK_INTERFACE_LIBRARIES ""
     LINK_FLAGS "${TPC_LINK_FLAGS}"
     COMPILE_DEFINITIONS "${XRD_COMPILE_DEFS}")
 
diff --git a/src/XrdTpc/XrdTpcMultistream.cc b/src/XrdTpc/XrdTpcMultistream.cc
index 973eae88636..aefc729383e 100644
--- a/src/XrdTpc/XrdTpcMultistream.cc
+++ b/src/XrdTpc/XrdTpcMultistream.cc
@@ -281,6 +281,9 @@ int TPCHandler::RunCurlWithStreamsImpl(XrdHttpExtReq &req, State &state,
         curl_handles.emplace_back(handles.back()->GetHandle());
     }
 
+    // Notify the packet marking manager that the transfer will start after this point
+  rec.pmarkManager.startTransfer(&req);
+
     // Create the multi-handle and add in the current transfer to it.
     MultiCurlHandler mch(handles, m_log);
     CURLM *multi_handle = mch.Get();
@@ -329,9 +332,14 @@ int TPCHandler::RunCurlWithStreamsImpl(XrdHttpExtReq &req, State &state,
             }
             int timeout = (transfer_start == last_advance_time) ? m_first_timeout : m_timeout;
             if (now > last_advance_time + timeout) {
+                const char *log_prefix = rec.log_prefix.c_str();
+                bool tpc_pull = strncmp("Pull", log_prefix, 4) == 0;
+
                 mch.SetErrorCode(10);
                 std::stringstream ss;
-                ss << "Transfer failed because no bytes have been received in " << timeout << " seconds.";
+                ss << "Transfer failed because no bytes have been "
+                   << (tpc_pull ? "received from the source (pull mode) in "
+                                : "transmitted to the destination (push mode) in ") << timeout << " seconds.";
                 mch.SetErrorMessage(ss.str());
                 break;
             }
@@ -347,6 +355,9 @@ int TPCHandler::RunCurlWithStreamsImpl(XrdHttpExtReq &req, State &state,
             break;
         }
 
+        rec.pmarkManager.beginPMarks();
+
+
         // Harvest any messages, looking for CURLMSG_DONE.
         CURLMsg *msg;
         do {
diff --git a/src/XrdTpc/XrdTpcPMarkManager.cc b/src/XrdTpc/XrdTpcPMarkManager.cc
new file mode 100644
index 00000000000..4d99bd6621a
--- /dev/null
+++ b/src/XrdTpc/XrdTpcPMarkManager.cc
@@ -0,0 +1,85 @@
+//------------------------------------------------------------------------------
+// This file is part of XrdTpcTPC
+//
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Cedric Caffy 
+// File Date: Oct 2023
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+
+#include 
+#include "XrdTpcPMarkManager.hh"
+
+namespace XrdTpc
+{
+PMarkManager::SocketInfo::SocketInfo(int fd, const struct sockaddr * sockP) {
+  netAddr.Set(sockP,fd);
+  client.addrInfo = static_cast(&netAddr);
+}
+
+PMarkManager::PMarkManager(XrdNetPMark *pmark) : mPmark(pmark), mTransferWillStart(false) {}
+
+void PMarkManager::addFd(int fd, const struct sockaddr * sockP) {
+  if(mPmark && mTransferWillStart && mReq->mSciTag >= 0) {
+    // The transfer will start and the packet marking has been configured, this socket must be registered for future packet marking
+    mSocketInfos.emplace(fd, sockP);
+  }
+}
+
+void PMarkManager::startTransfer(XrdHttpExtReq * req) {
+  mReq = req;
+  mTransferWillStart = true;
+}
+
+void PMarkManager::beginPMarks() {
+  if(mSocketInfos.empty()) {
+    return;
+  }
+
+  if(mPmarkHandles.empty()) {
+    // Create the first pmark handle
+    std::stringstream ss;
+    ss << "scitag.flow=" << mReq->mSciTag;
+    SocketInfo & sockInfo = mSocketInfos.front();
+    auto pmark = mPmark->Begin(sockInfo.client, mReq->resource.c_str(), ss.str().c_str(), "http-tpc");
+    if(!pmark) {
+      return;
+    }
+    mPmarkHandles.emplace(sockInfo.client.addrInfo->SockFD(),std::unique_ptr(pmark));
+    mSocketInfos.pop();
+  }
+
+  auto pmarkHandleItor = mPmarkHandles.begin();
+  while(!mSocketInfos.empty()) {
+    SocketInfo & sockInfo = mSocketInfos.front();
+    auto pmark = mPmark->Begin(*sockInfo.client.addrInfo, *(pmarkHandleItor->second), nullptr);
+    if (!pmark) {
+      // The packet marking handle could not be created from the first handle, let's retry next time
+      break;
+    }
+
+    int fd = sockInfo.client.addrInfo->SockFD();
+    mPmarkHandles.emplace(fd, std::move(std::unique_ptr(pmark)));
+    mSocketInfos.pop();
+  }
+}
+
+void PMarkManager::endPmark(int fd) {
+  // We need to delete the PMark handle associated to the fd passed in parameter
+  // we just look for it and reset the unique_ptr to nullptr to trigger the PMark handle deletion
+  mPmarkHandles.erase(fd);
+}
+} // namespace XrdTpc
diff --git a/src/XrdTpc/XrdTpcPMarkManager.hh b/src/XrdTpc/XrdTpcPMarkManager.hh
new file mode 100644
index 00000000000..938de5c62d8
--- /dev/null
+++ b/src/XrdTpc/XrdTpcPMarkManager.hh
@@ -0,0 +1,113 @@
+//------------------------------------------------------------------------------
+// This file is part of XrdTpcTPC
+//
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Cedric Caffy 
+// File Date: Oct 2023
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+#ifndef XROOTD_PMARKMANAGER_HH
+#define XROOTD_PMARKMANAGER_HH
+
+#include "XrdNet/XrdNetPMark.hh"
+#include "XrdSec/XrdSecEntity.hh"
+#include "XrdNet/XrdNetAddrInfo.hh"
+#include "XrdNet/XrdNetAddr.hh"
+#include "XrdHttp/XrdHttpExtHandler.hh"
+
+#include 
+#include 
+#include 
+
+/**
+ * This class will manage packet marking handles for TPC transfers
+ *
+ * Each time a socket will be opened by curl (via the opensocket_callback), the manager
+ * will register the related information to the socket.
+ *
+ * Once the transfer will start we will start the packet marking by creating XrdNetPMark::Handle
+ * objects from the socket information previously registered.
+ *
+ * In the case of multi-stream HTTP TPC transfers, a packet marking handle will be created for each stream.
+ * The first one will be created as a basic one. The other will be created using the first packet marking handle as a basis.
+ */
+namespace XrdTpc
+{
+class PMarkManager {
+public:
+
+  /**
+   * This class allows to create and keep a XrdSecEntity object
+   * from the socket file descriptor and address
+   * Everything is done on the constructor
+   *
+   * These infos will be used later on when we create new PMark handles
+   */
+  class SocketInfo {
+  public:
+    SocketInfo(int fd, const struct sockaddr * sockP);
+    XrdNetAddr netAddr;
+    XrdSecEntity client;
+  };
+
+  PMarkManager(XrdNetPMark * pmark);
+  /**
+   * Add the connected socket information that will be used for packet marking to this manager class
+   * Note: these info will only be added if startTransfer(...) has been called. It allows
+   * to ensure that the connection will be related to the data transfers and not for anything else. We only want
+   * to mark the traffic of the transfers.
+   * @param fd the socket file descriptor
+   * @param sockP the structure describing the address of the socket
+   */
+  void addFd(int fd, const struct sockaddr * sockP);
+
+  /**
+   * Calling this function will indicate that the connections that will happen will be related to the
+   * data transfer. The addFd(...) function will then register any socket that is created after this function
+   * will be called.
+   * @param req the request object that will be used later on to get some information about the transfer
+   */
+  void startTransfer(XrdHttpExtReq * req);
+  /**
+   * Creates the different packet marking handles allowing to mark the transfer packets
+   *
+   * Call this after the curl_multi_perform() has been called.
+   */
+  void beginPMarks();
+
+  /**
+   * This function deletes the PMark handle associated to the fd passed in parameter
+   * Use this before closing the associated socket! Otherwise the information contained in the firefly
+   * (e.g sent bytes or received bytes) will have values equal to 0.
+   * @param fd the fd of the socket to be closed
+   */
+  void endPmark(int fd);
+
+  virtual ~PMarkManager() = default;
+private:
+  // The queue of socket information from which we will create the packet marking handles
+  std::queue mSocketInfos;
+  // The map of socket FD and packet marking handles
+  std::map> mPmarkHandles;
+  // The instance of the packet marking functionality
+  XrdNetPMark * mPmark;
+  // Is true when startTransfer(...) has been called
+  bool mTransferWillStart;
+  // The XrdHttpTPC request information
+  XrdHttpExtReq * mReq;
+};
+} // namespace XrdTpc
+
+#endif //XROOTD_PMARKMANAGER_HH
diff --git a/src/XrdTpc/XrdTpcTPC.cc b/src/XrdTpc/XrdTpcTPC.cc
index 6d327806bcd..bd35d8ce79e 100644
--- a/src/XrdTpc/XrdTpcTPC.cc
+++ b/src/XrdTpc/XrdTpcTPC.cc
@@ -119,18 +119,34 @@ int TPCHandler::opensocket_callback(void *clientp,
                                     curlsocktype purpose,
                                     struct curl_sockaddr *aInfo)
 {
-// See what kind of address will be used to connect
-//
-if (purpose == CURLSOCKTYPE_IPCXN && clientp)
-   {XrdNetAddr thePeer(&(aInfo->addr));
-    ((TPCLogRecord *)clientp)->isIPv6 =  (thePeer.isIPType(XrdNetAddrInfo::IPv6)
-                                      && !thePeer.isMapped());
-   }
+  //Return a socket file descriptor (note the clo_exec flag will be set).
+  int fd = XrdSysFD_Socket(aInfo->family, aInfo->socktype, aInfo->protocol);
+  // See what kind of address will be used to connect
+  //
+  if(fd < 0) {
+    return CURL_SOCKET_BAD;
+  }
+  TPCLogRecord * rec = (TPCLogRecord *)clientp;
+  if (purpose == CURLSOCKTYPE_IPCXN && clientp)
+  {XrdNetAddr thePeer(&(aInfo->addr));
+    rec->isIPv6 =  (thePeer.isIPType(XrdNetAddrInfo::IPv6)
+                    && !thePeer.isMapped());
+    // Register the socket to the packet marking manager
+    rec->pmarkManager.addFd(fd,&aInfo->addr);
+  }
+
+  return fd;
+}
 
-// Return a socket file descriptor (note the clo_exec flag will be set).
-//
-   int fd = XrdSysFD_Socket(aInfo->family, aInfo->socktype, aInfo->protocol);
-   return (fd >= 0 ? fd : CURL_SOCKET_BAD);
+int TPCHandler::closesocket_callback(void *clientp, curl_socket_t fd) {
+  TPCLogRecord * rec = (TPCLogRecord *)clientp;
+
+  // Destroy the PMark handle associated to the file descriptor before closing it.
+  // Otherwise, we would lose the socket usage information if the socket is closed before
+  // the PMark handle is closed.
+  rec->pmarkManager.endPmark(fd);
+
+  return close(fd);
 }
 
 /******************************************************************************/
@@ -634,9 +650,14 @@ int TPCHandler::RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, State &state,
             }
             int timeout = (transfer_start == last_advance_time) ? m_first_timeout : m_timeout;
             if (now > last_advance_time + timeout) {
+                const char *log_prefix = rec.log_prefix.c_str();
+                bool tpc_pull = strncmp("Pull", log_prefix, 4) == 0;
+
                 state.SetErrorCode(10);
                 std::stringstream ss;
-                ss << "Transfer failed because no bytes have been received in " << timeout << " seconds.";
+                ss << "Transfer failed because no bytes have been "
+                   << (tpc_pull ? "received from the source (pull mode) in "
+                                : "transmitted to the destination (push mode) in ") << timeout << " seconds.";
                 state.SetErrorMessage(ss.str());
                 curl_multi_remove_handle(multi_handle, curl);
                 curl_multi_cleanup(multi_handle);
@@ -644,6 +665,8 @@ int TPCHandler::RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, State &state,
             }
             last_marker = now;
         }
+        // The transfer will start after this point, notify the packet marking manager
+      rec.pmarkManager.startTransfer(&req);
         mres = curl_multi_perform(multi_handle, &running_handles);
         if (mres == CURLM_CALL_MULTI_PERFORM) {
             // curl_multi_perform should be called again immediately.  On newer
@@ -654,6 +677,8 @@ int TPCHandler::RunCurlWithUpdates(CURL *curl, XrdHttpExtReq &req, State &state,
         } else if (running_handles == 0) {
             break;
         }
+
+        rec.pmarkManager.beginPMarks();
         //printf("There are %d running handles\n", running_handles);
 
         // Harvest any messages, looking for CURLMSG_DONE.
@@ -828,7 +853,7 @@ int TPCHandler::RunCurlBasic(CURL *curl, XrdHttpExtReq &req, State &state,
 /******************************************************************************/
   
 int TPCHandler::ProcessPushReq(const std::string & resource, XrdHttpExtReq &req) {
-    TPCLogRecord rec;
+    TPCLogRecord rec(req.pmark);
     rec.log_prefix = "PushRequest";
     rec.local = req.resource;
     rec.remote = resource;
@@ -849,6 +874,8 @@ int TPCHandler::ProcessPushReq(const std::string & resource, XrdHttpExtReq &req)
 //  curl_easy_setopt(curl, CURLOPT_SOCKOPTFUNCTION, sockopt_setcloexec_callback);
     curl_easy_setopt(curl, CURLOPT_OPENSOCKETFUNCTION, opensocket_callback);
     curl_easy_setopt(curl, CURLOPT_OPENSOCKETDATA, &rec);
+    curl_easy_setopt(curl, CURLOPT_CLOSESOCKETFUNCTION, closesocket_callback);
+    curl_easy_setopt(curl, CURLOPT_CLOSESOCKETDATA, &rec);
     auto query_header = req.headers.find("xrd-http-fullresource");
     std::string redirect_resource = req.resource;
     if (query_header != req.headers.end()) {
@@ -908,7 +935,7 @@ int TPCHandler::ProcessPushReq(const std::string & resource, XrdHttpExtReq &req)
 /******************************************************************************/
   
 int TPCHandler::ProcessPullReq(const std::string &resource, XrdHttpExtReq &req) {
-    TPCLogRecord rec;
+    TPCLogRecord rec(req.pmark);
     rec.log_prefix = "PullRequest";
     rec.local = req.resource;
     rec.remote = resource;
@@ -929,6 +956,8 @@ int TPCHandler::ProcessPullReq(const std::string &resource, XrdHttpExtReq &req)
 //  curl_easy_setopt(curl,CURLOPT_SOCKOPTFUNCTION,sockopt_setcloexec_callback);
     curl_easy_setopt(curl, CURLOPT_OPENSOCKETFUNCTION, opensocket_callback);
     curl_easy_setopt(curl, CURLOPT_OPENSOCKETDATA, &rec);
+    curl_easy_setopt(curl, CURLOPT_CLOSESOCKETFUNCTION, closesocket_callback);
+    curl_easy_setopt(curl, CURLOPT_CLOSESOCKETDATA, &rec);
     std::unique_ptr fh(m_sfs->newFile(name, m_monid++));
     if (!fh.get()) {
             char msg[] = "Failed to initialize internal transfer file handle";
@@ -957,7 +986,7 @@ int TPCHandler::ProcessPullReq(const std::string &resource, XrdHttpExtReq &req)
             }
             if (stream_req < 0 || stream_req > 100) {
                 char msg[] = "Invalid request for number of streams";
-                rec.status = 500;
+                rec.status = 400;
                 logTransferEvent(LogMask::Info, rec, "INVALID_REQUEST", msg);
                 return req.SendSimpleResp(rec.status, NULL, NULL, msg, 0);
             }
diff --git a/src/XrdTpc/XrdTpcTPC.hh b/src/XrdTpc/XrdTpcTPC.hh
index 5b8090d5bf9..76f600c58c9 100644
--- a/src/XrdTpc/XrdTpcTPC.hh
+++ b/src/XrdTpc/XrdTpcTPC.hh
@@ -10,6 +10,7 @@
 #include "XrdHttp/XrdHttpUtils.hh"
 
 #include "XrdTls/XrdTlsTempCA.hh"
+#include "XrdTpcPMarkManager.hh"
 
 #include 
 
@@ -55,10 +56,12 @@ private:
                                    curlsocktype purpose,
                                    struct curl_sockaddr *address);
 
+    static int closesocket_callback(void *clientp, curl_socket_t fd);
+
     struct TPCLogRecord {
 
-        TPCLogRecord() : bytes_transferred( -1 ), status( -1 ),
-                         tpc_status(-1), streams( 1 ), isIPv6(false)
+        TPCLogRecord(XrdNetPMark * pmark) : bytes_transferred( -1 ), status( -1 ),
+                         tpc_status(-1), streams( 1 ), isIPv6(false), pmarkManager(pmark)
         {
          gettimeofday(&begT, 0); // Set effective start time
         }
@@ -76,6 +79,7 @@ private:
         int tpc_status;
         unsigned int streams;
         bool isIPv6;
+        XrdTpc::PMarkManager pmarkManager;
     };
 
     int ProcessOptionsReq(XrdHttpExtReq &req);
diff --git a/src/XrdUtils.cmake b/src/XrdUtils.cmake
index 05b4146d92b..8621c3b974f 100644
--- a/src/XrdUtils.cmake
+++ b/src/XrdUtils.cmake
@@ -1,5 +1,4 @@
 
-include( XRootDCommon )
 
 #-------------------------------------------------------------------------------
 # Shared library version
@@ -83,7 +82,7 @@ set ( XrdCryptoSources
       XrdCrypto/XrdCryptoX509Chain.cc           XrdCrypto/XrdCryptoX509Chain.hh
       XrdCrypto/XrdCryptosslRSA.cc              XrdCrypto/XrdCryptosslRSA.hh
       XrdCrypto/XrdCryptoRSA.cc                 XrdCrypto/XrdCryptoRSA.hh
-      XrdCrypto/XrdCryptosslgsiAux.cc           XrdCrypto/XrdCryptosslgsiAux.hh
+      XrdCrypto/XrdCryptosslgsiAux.cc
       XrdCrypto/XrdCryptosslX509Req.cc          XrdCrypto/XrdCryptosslX509Req.hh
       XrdCrypto/XrdCryptoX509Req.cc             XrdCrypto/XrdCryptoX509Req.hh
       XrdCrypto/XrdCryptoAux.cc                 XrdCrypto/XrdCryptoAux.hh
@@ -266,6 +265,7 @@ set ( XrdSecSources
   XrdSec/XrdSecEntityAttr.cc       XrdSec/XrdSecEntityAttr.hh
   XrdSec/XrdSecEntityXtra.cc       XrdSec/XrdSecEntityXtra.hh
   XrdSec/XrdSecLoadSecurity.cc     XrdSec/XrdSecLoadSecurity.hh
+                                   XrdSec/XrdSecMonitor.hh
   XrdSecsss/XrdSecsssCon.cc        XrdSecsss/XrdSecsssCon.hh
   XrdSecsss/XrdSecsssEnt.cc        XrdSecsss/XrdSecsssEnt.hh
   XrdSecsss/XrdSecsssID.cc         XrdSecsss/XrdSecsssID.hh
@@ -291,6 +291,7 @@ add_library(
 
 target_link_libraries(
   XrdUtils
+  PRIVATE
   OpenSSL::SSL
   ${CMAKE_THREAD_LIBS_INIT}
   ${CMAKE_DL_LIBS}
@@ -300,7 +301,7 @@ target_link_libraries(
 
 if ( SYSTEMD_FOUND )
    target_link_libraries(
-     XrdUtils
+     XrdUtils PRIVATE
      ${SYSTEMD_LIBRARIES}
    )
 endif()
@@ -310,9 +311,7 @@ set_target_properties(
   PROPERTIES
   BUILD_RPATH ${CMAKE_CURRENT_BINARY_DIR}
   VERSION   ${XRD_UTILS_VERSION}
-  SOVERSION ${XRD_UTILS_SOVERSION}
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
+  SOVERSION ${XRD_UTILS_SOVERSION} )
 
 #-------------------------------------------------------------------------------
 # Install
diff --git a/src/XrdVersion.hh.in b/src/XrdVersion.hh.in
index b764bdbbf9d..5bd0236443f 100644
--- a/src/XrdVersion.hh.in
+++ b/src/XrdVersion.hh.in
@@ -25,20 +25,17 @@
 /* specific prior written permission of the institution or contributor.       */
 /******************************************************************************/
 
-// this file is automatically updated by the genversion.sh script
-// if you touch anything make sure that it still works
-
 #ifndef __XRD_VERSION_H__
 #define __XRD_VERSION_H__
 
-#define XrdVERSION  "unknown"
+#define XrdVERSION  "@XRootD_VERSION_STRING@"
 
 // Numeric representation of the version tag
 // The format for the released code is: xyyzz, where: x is the major version,
 // y is the minor version and zz is the bugfix revision number
 // For the non-released code the value is 1000000
 #define XrdVNUMUNK  1000000
-#define XrdVNUMBER  1000000
+#define XrdVNUMBER  @XRootD_VERSION_NUMBER@
 
 #if XrdDEBUG
 #define XrdVSTRING XrdVERSION "_dbg"
diff --git a/src/XrdVoms.cmake b/src/XrdVoms.cmake
index 1951a376eb0..55cdf2a3b2d 100644
--- a/src/XrdVoms.cmake
+++ b/src/XrdVoms.cmake
@@ -3,8 +3,6 @@
 # The XrdSecgsiVOMS library
 #-------------------------------------------------------------------------------
 
-include_directories( ${VOMS_INCLUDE_DIR} )
-
 set( LIB_XRD_VOMS             XrdVoms-${PLUGIN_VERSION} )
 set( LIB_XRD_SEC_GSI_VOMS     XrdSecgsiVOMS-${PLUGIN_VERSION} )
 set( LIB_XRD_HTTP_VOMS        XrdHttpVOMS-${PLUGIN_VERSION} )
@@ -19,14 +17,11 @@ add_library(
 
 target_link_libraries(
    ${LIB_XRD_VOMS}
+   PRIVATE
    XrdCrypto
    ${VOMS_LIBRARIES} )
 
-set_target_properties(
-   ${LIB_XRD_VOMS}
-   PROPERTIES
-   INTERFACE_LINK_LIBRARIES ""
-   LINK_INTERFACE_LIBRARIES "" )
+target_include_directories( ${LIB_XRD_VOMS} PRIVATE ${VOMS_INCLUDE_DIR} )
 
 #-------------------------------------------------------------------------------
 # Install
@@ -36,11 +31,6 @@ install(
    RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} )
 
-install(
-  FILES
-  ${PROJECT_SOURCE_DIR}/docs/man/libXrdVoms.1
-  DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 )
-
 install(
   CODE "
     EXECUTE_PROCESS(
@@ -52,9 +42,3 @@ install(
     EXECUTE_PROCESS(
       COMMAND ln -sf lib${LIB_XRD_VOMS}.so lib${LIB_XRD_HTTP_VOMS}.so
       WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR} )" )
-
-install(
-  CODE "
-    EXECUTE_PROCESS(
-      COMMAND ln -sf libXrdVoms.1 libXrdSecgsiVOMS.1
-      WORKING_DIRECTORY \$ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_MANDIR}/man1 )" )
diff --git a/src/XrdVoms/XrdVomsFun.cc b/src/XrdVoms/XrdVomsFun.cc
index a4ced43ba5c..2b2afa7327a 100644
--- a/src/XrdVoms/XrdVomsFun.cc
+++ b/src/XrdVoms/XrdVomsFun.cc
@@ -84,7 +84,7 @@
       while ((sp = a.find(' ', sp+1)) != STR_NPOS) { a[sp] = '\t'; } \
    }
 
-#define FATAL(x) {cerr <<"VomsFun: "< &ent
 
 
 std::string
-XrdVomsMapfile::Map(const std::vector &fqan)
+XrdVomsMapfile::Map(const std::vector &fqan)
 {
     decltype(m_entries) entries = m_entries;
     if (!entries) {return "";}
diff --git a/src/XrdVoms/XrdVomsTrace.hh b/src/XrdVoms/XrdVomsTrace.hh
index 803c73f929b..15f0e426c89 100644
--- a/src/XrdVoms/XrdVomsTrace.hh
+++ b/src/XrdVoms/XrdVomsTrace.hh
@@ -32,8 +32,8 @@
 
 #ifndef NODEBUG
 
-#define PRINT(y)    if (gDebug) {cerr <traceBeg() <<" XrdVoms"\
-                                      <traceBeg() <<" XrdVoms"\
+                                      <traceEnd();}
 #define DEBUG(y)    if (gDebug > 1) {PRINT(y)}
 #define EPNAME(x)   static const char *epname = x;
diff --git a/src/XrdXml.cmake b/src/XrdXml.cmake
index 6d87414c18d..d4de72675dd 100644
--- a/src/XrdXml.cmake
+++ b/src/XrdXml.cmake
@@ -1,5 +1,4 @@
 
-include( XRootDCommon )
 
 if ( TINYXML_FOUND )
    set( TINYXML_FILES "" )
@@ -45,6 +44,7 @@ add_library(
 
 target_link_libraries(
   XrdXml
+  PRIVATE
   XrdUtils
   ${TINYXML_LIBRARIES}
   ${XRDXML2_LIBRARIES}
@@ -54,9 +54,7 @@ set_target_properties(
   XrdXml
   PROPERTIES
   VERSION   ${XRD_XML_VERSION}
-  SOVERSION ${XRD_XML_SOVERSION}
-  INTERFACE_LINK_LIBRARIES ""
-  LINK_INTERFACE_LIBRARIES "" )
+  SOVERSION ${XRD_XML_SOVERSION} )
 
 if ( TINYXML_FOUND )
    target_include_directories( XrdXml PRIVATE ${TINYXML_INCLUDE_DIR} )
diff --git a/src/XrdXrootd/XrdXrootdAioBuff.hh b/src/XrdXrootd/XrdXrootdAioBuff.hh
index 2d2209741b4..174433dd1e3 100644
--- a/src/XrdXrootd/XrdXrootdAioBuff.hh
+++ b/src/XrdXrootd/XrdXrootdAioBuff.hh
@@ -48,7 +48,7 @@ XrdXrootdAioBuff*       Alloc(XrdXrootdAioTask *arp);
 
         void            doneWrite() override;
 
-virtual void            Recycle();
+virtual void            Recycle() override;
 
 XrdXrootdAioBuff*       next;
 
diff --git a/src/XrdXrootd/XrdXrootdConfig.cc b/src/XrdXrootd/XrdXrootdConfig.cc
index 38e42458019..702e952bbfa 100644
--- a/src/XrdXrootd/XrdXrootdConfig.cc
+++ b/src/XrdXrootd/XrdXrootdConfig.cc
@@ -219,6 +219,15 @@ int XrdXrootdProtocol::Configure(char *parms, XrdProtocol_Config *pi)
 //
    n = (pi->theEnv ? pi->theEnv->GetInt("MaxBuffSize") : 0);
    maxTransz = maxBuffsz = (n ? n : BPool->MaxSize());
+   maxReadv_ior = maxTransz-(int)sizeof(readahead_list);
+
+// Export the readv_ior_max and readv_iov_max values
+//
+  {char buff[256];
+   snprintf(buff, sizeof(buff), "%d,%d",  maxReadv_ior, XrdProto::maxRvecsz);
+   XrdOucEnv::Export("XRD_READV_LIMITS", buff);
+  }
+
    memset(Route, 0, sizeof(Route));
 
 // Now process and configuration parameters
diff --git a/src/XrdXrootd/XrdXrootdFile.cc b/src/XrdXrootd/XrdXrootdFile.cc
index f7fb1a62465..deab025b376 100644
--- a/src/XrdXrootd/XrdXrootdFile.cc
+++ b/src/XrdXrootd/XrdXrootdFile.cc
@@ -170,7 +170,10 @@ void XrdXrootdFile::Ref(int num)
    fileMutex.Lock();
    refCount += num;
    TRACEI(FSAIO,"File::Ref="<Post();
+   if (num < 0 && syncWait && refCount <= 0)
+      {syncWait->Post();
+       syncWait = nullptr;
+      }
    fileMutex.UnLock();
 }
 
@@ -215,6 +218,7 @@ int XrdXrootdFileTable::Add(XrdXrootdFile *fp)
           else {i -= XRD_FTABSIZE;
                 if (XTab && i < XTnum) fP = &XTab[i];
                    else fP = 0;
+                i += XRD_FTABSIZE;
                }
        if (fP && *fP == heldSpotP)
           {*fP = fp;
diff --git a/src/XrdXrootd/XrdXrootdMonData.hh b/src/XrdXrootd/XrdXrootdMonData.hh
index 29dfda85645..1c98a6b4707 100644
--- a/src/XrdXrootd/XrdXrootdMonData.hh
+++ b/src/XrdXrootd/XrdXrootdMonData.hh
@@ -110,6 +110,7 @@ const kXR_char XROOTD_MON_MAPPURG       = 'p';
 const kXR_char XROOTD_MON_MAPREDR       = 'r';
 const kXR_char XROOTD_MON_MAPSTAG       = 's'; // Internal use only!
 const kXR_char XROOTD_MON_MAPTRCE       = 't';
+const kXR_char XROOTD_MON_MAPTOKN       = 'T';
 const kXR_char XROOTD_MON_MAPUSER       = 'u';
 const kXR_char XROOTD_MON_MAPUEAC       = 'U'; // User experiment/activity
 const kXR_char XROOTD_MON_MAPXFER       = 'x';
diff --git a/src/XrdXrootd/XrdXrootdMonitor.cc b/src/XrdXrootd/XrdXrootdMonitor.cc
index d0f11739296..08dc60493d7 100644
--- a/src/XrdXrootd/XrdXrootdMonitor.cc
+++ b/src/XrdXrootd/XrdXrootdMonitor.cc
@@ -286,29 +286,37 @@ void XrdXrootdMonitor::User::Enable()
 /******************************************************************************/
 /*      X r d X r o o t d M o n i t o r : : U s e r : : R e g i s t e r       */
 /******************************************************************************/
-  
+
 void XrdXrootdMonitor::User::Register(const char *Uname, 
                                       const char *Hname,
-                                      const char *Pname)
+                                      const char *Pname, unsigned int xSID)
 {
-   const char *colonP, *atP;
-   char  uBuff[1024], *uBP;
-   int n;
+#ifndef NODEBUG
+    const char *TraceID = "Monitor";
+#endif
+   char *dotP, *colonP, *atP;
+   char  uBuff[1024], tBuff[1024], sBuff[64];
+
+// Decode the user name as a.b:c@d and remap it for monitoring as
+// /a.{b|xSID}:@host
+//
+   snprintf(tBuff, sizeof(tBuff), "%s", Uname);
+   if ((dotP = index(tBuff, '.')) && (colonP = index(dotP+1, ':')) && 
+       (atP = index(colonP+1, '@')))
+      {*dotP = 0; *colonP = 0; *atP = 0;
+       if (xSID)  
+          {snprintf(sBuff, sizeof(sBuff), " %u", xSID);
+           dotP = sBuff;
+          }
 
-// The identification always starts with the protocol being used
-//
-   n = sprintf(uBuff, "%s/", Pname);
-   uBP = uBuff + n;
+       int n = snprintf(uBuff, sizeof(uBuff), "%s/%s.%s:%s@%s", Pname, tBuff,
+                        dotP+1, kySID, atP+1);
 
-// Decode the user name as a.b:c@d
-//
-   if ((colonP = index(Uname, ':')) && (atP = index(colonP+1, '@')))
-      {n = colonP - Uname + 1;
-       strncpy(uBP, Uname, n);
-       strcpy(uBP+n, kySID);
-       n += kySIDSZ; *(uBP+n) = '@'; n++;
-       strcpy(uBP+n, Hname);
-      } else strcpy(uBP, Uname);
+       if (n < 0 || n >= (int) sizeof(uBuff))
+         TRACE(LOGIN, "Login ID was truncated: " << uBuff);
+
+       if (xSID) {TRACE(LOGIN,"Register remap "< "<
 #include 
 
+#include "XrdSec/XrdSecMonitor.hh"
 #include "XrdSys/XrdSysPthread.hh"
 #include "XrdXrootd/XrdXrootdMonData.hh"
 #include "XProtocol/XPtypes.hh"
@@ -167,7 +168,7 @@ static  Hello     *First;
 
 /******************************************************************************/
 
-class  User
+class  User : public XrdSecMonitor
 {
 public:
 
@@ -210,13 +211,15 @@ inline kXR_unt32   MapPath(const char *Path)
                           }
 
        void        Register(const char *Uname, const char *Hname,
-                            const char *Pname);
+                            const char *Pname, unsigned int xSID=0);
 
        void        Report(const char *Info)
                          {Did=XrdXrootdMonitor::Map(XROOTD_MON_MAPUSER,*this,Info);}
 
        void        Report(int eCode, int aCode);
 
+       bool        Report(WhatInfo infoT, const char *info) override;
+
 inline int         Ready()  {return XrdXrootdMonitor::monACTIVE;}
 
        User() : Agent(0), Did(0), Iops(0), Fops(0), Len(0), Name(0) {}
diff --git a/src/XrdXrootd/XrdXrootdProtocol.cc b/src/XrdXrootd/XrdXrootdProtocol.cc
index ddc872fd7e0..e0e50e75d7d 100644
--- a/src/XrdXrootd/XrdXrootdProtocol.cc
+++ b/src/XrdXrootd/XrdXrootdProtocol.cc
@@ -113,6 +113,8 @@ XrdNetSocket         *XrdXrootdProtocol::AdminSock= 0;
 int                   XrdXrootdProtocol::hcMax        = 28657; // const for now
 int                   XrdXrootdProtocol::maxBuffsz;
 int                   XrdXrootdProtocol::maxTransz    = 262144; // 256KB
+int                   XrdXrootdProtocol::maxReadv_ior =
+                      XrdXrootdProtocol::maxTransz-(int)sizeof(readahead_list);
 int                   XrdXrootdProtocol::as_maxperlnk = 8;   // Max ops per link
 int                   XrdXrootdProtocol::as_maxperreq = 8;   // Max ops per request
 int                   XrdXrootdProtocol::as_maxpersrv = 4096;// Max ops per server
@@ -1501,7 +1503,7 @@ void XrdXrootdProtocol::Reset()
    doTLS              = tlsNot; // Assume client is not capable. This will be
    ableTLS            = false;  // resolved during the kXR_protocol interchange.
    isTLS              = false;  // Made true when link converted to TLS
-   linkAioReq         = {0};
+   linkAioReq         = 0;
    pioFree = pioFirst = pioLast = 0;
    isActive = isLinkWT= isNOP = isDead = false;
    sigNeed = sigHere = sigRead = false;
diff --git a/src/XrdXrootd/XrdXrootdProtocol.hh b/src/XrdXrootd/XrdXrootdProtocol.hh
index 162a30e5925..50a23046ca6 100644
--- a/src/XrdXrootd/XrdXrootdProtocol.hh
+++ b/src/XrdXrootd/XrdXrootdProtocol.hh
@@ -163,11 +163,11 @@ public:
 
 static char         *Buffer(XrdSfsXioHandle h, int *bsz); // XrdSfsXio
 
-XrdSfsXioHandle      Claim(const char *buff, int datasz, int minasz=0);// XrdSfsXio
+XrdSfsXioHandle      Claim(const char *buff, int datasz, int minasz=0) override;// XrdSfsXio
 
 static int           Configure(char *parms, XrdProtocol_Config *pi);
 
-       void          DoIt() {(*this.*Resume)();}
+       void          DoIt() override {(*this.*Resume)();}
 
        int           do_WriteSpan();
 
@@ -181,29 +181,29 @@ static int           Configure(char *parms, XrdProtocol_Config *pi);
 
        int           getPathID() {return PathID;}
 
-       XrdProtocol  *Match(XrdLink *lp);
+       XrdProtocol  *Match(XrdLink *lp) override;
 
-       int           Process(XrdLink *lp); //  Sync: Job->Link.DoIt->Process
+       int           Process(XrdLink *lp) override; //  Sync: Job->Link.DoIt->Process
 
        int           Process2();
 
        int           ProcSig();
 
-       void          Recycle(XrdLink *lp, int consec, const char *reason);
+       void          Recycle(XrdLink *lp, int consec, const char *reason) override;
 
 static void          Reclaim(XrdSfsXioHandle h); // XrdSfsXio
 
-       int           SendFile(int fildes); // XrdSfsDio
+       int           SendFile(int fildes) override; // XrdSfsDio
 
-       int           SendFile(XrdOucSFVec *sfvec, int sfvnum); // XrdSfsDio
+       int           SendFile(XrdOucSFVec *sfvec, int sfvnum) override; // XrdSfsDio
 
-       void          SetFD(int fildes); // XrdSfsDio
+       void          SetFD(int fildes) override; // XrdSfsDio
 
-       int           Stats(char *buff, int blen, int do_sync=0);
+       int           Stats(char *buff, int blen, int do_sync=0) override;
 
        void          StreamNOP();
 
-XrdSfsXioHandle      Swap(const char *buff, XrdSfsXioHandle h=0); // XrdSfsXio
+XrdSfsXioHandle      Swap(const char *buff, XrdSfsXioHandle h=0) override; // XrdSfsXio
 
 XrdXrootdProtocol   *VerifyStream(int &rc, int pID, bool lok=true);
 
@@ -285,6 +285,7 @@ enum RD_func {RD_chmod = 0, RD_chksum,  RD_dirlist, RD_locate, RD_mkdir,
        int   do_Rm();
        int   do_Rmdir();
        int   do_Set();
+       int   do_Set_Cache(XrdOucTokenizer &setargs);
        int   do_Set_Mon(XrdOucTokenizer &setargs);
        int   do_Stat();
        int   do_Statx();
@@ -470,6 +471,7 @@ static char   tlsNot;    // TLS requirements for incapable clients
 //
 static int                 maxBuffsz;    // Maximum buffer size we can have
 static int                 maxTransz;    // Maximum transfer size we can have
+static int                 maxReadv_ior; // Maximum readv element length
 
 // Statistical area
 //
diff --git a/src/XrdXrootd/XrdXrootdTpcMon.cc b/src/XrdXrootd/XrdXrootdTpcMon.cc
index fe41583fc89..e95ec709795 100644
--- a/src/XrdXrootd/XrdXrootdTpcMon.cc
+++ b/src/XrdXrootd/XrdXrootdTpcMon.cc
@@ -44,7 +44,7 @@ namespace
 const char *json_fmt = "{\"TPC\":\"%s\",\"Client\":\"%s\","
 "\"Xeq\":{\"Beg\":\"%s\",\"End\":\"%s\",\"RC\":%d,\"Strm\":%u,\"Type\":\"%s\","
         "\"IPv\":%c},"
-"\"Src\":\"%s\",\"Dst\":\"%s\",\"Size\":%d}";
+"\"Src\":\"%s\",\"Dst\":\"%s\",\"Size\":%zu}";
 
 const char *urlFMT = "";
 
diff --git a/src/XrdXrootd/XrdXrootdXeq.cc b/src/XrdXrootd/XrdXrootdXeq.cc
index 1ca6cf79d83..5efe8df9162 100644
--- a/src/XrdXrootd/XrdXrootdXeq.cc
+++ b/src/XrdXrootd/XrdXrootdXeq.cc
@@ -1109,10 +1109,11 @@ int XrdXrootdProtocol::do_Login()
 // Allocate a monitoring object, if needed for this connection
 //
    if (Monitor.Ready())
-      {Monitor.Register(Link->ID, Link->Host(), "xroot");
+      {Monitor.Register(Link->ID, Link->Host(), "xroot", mySID);
        if (Monitor.Logins() && (!Monitor.Auths() || !(Status & XRD_NEED_AUTH)))
           {Monitor.Report(Entity.moninfo);
            if (Entity.moninfo) {free(Entity.moninfo); Entity.moninfo = 0;}
+           Entity.secMon = &Monitor;
           }
       }
 
@@ -2017,7 +2018,7 @@ int XrdXrootdProtocol::do_Qconf()
             bp += n; bleft -= n;
            }
    else if (!strcmp("readv_ior_max", val))
-           {n = snprintf(bp,bleft,"%d\n",maxTransz-(int)sizeof(readahead_list));
+           {n = snprintf(bp,bleft,"%d\n",maxReadv_ior);
             bp += n; bleft -= n;
            }
    else if (!strcmp("readv_iov_max", val)) 
@@ -2551,7 +2552,7 @@ int XrdXrootdProtocol::do_ReadV()
 // partial elements.
 //
    rdVecNum = rdVecLen / sizeof(readahead_list);
-   if ( (rdVecLen <= 0) || (rdVecNum*hdrSZ != rdVecLen) )
+   if ( (rdVecNum <= 0) || (rdVecNum*hdrSZ != rdVecLen) )
       return Response.Send(kXR_ArgInvalid, "Read vector is invalid");
 
 // Make sure that we can copy the read vector to our local stack. We must impose 
@@ -2570,7 +2571,7 @@ int XrdXrootdProtocol::do_ReadV()
 // to copy the read ahead list to our readv vector for later processing.
 //
    raVec = (readahead_list *)argp->buff;
-   totSZ = rdVecLen; Quantum = maxTransz - hdrSZ;
+   totSZ = rdVecLen; Quantum = maxReadv_ior;
    for (i = 0; i < rdVecNum; i++) 
        {totSZ += (rdVec[i].size = ntohl(raVec[i].rlen));
         if (rdVec[i].size < 0)       return Response.Send(kXR_ArgInvalid,
@@ -2770,12 +2771,57 @@ int XrdXrootdProtocol::do_Set()
             return Response.Send();
            }
    else if (!strcmp("monitor", val)) return do_Set_Mon(setargs);
+   else if (!strcmp("cache",   val)) return do_Set_Cache(setargs);
 
 // All done
 //
    return Response.Send(kXR_ArgInvalid, "invalid set parameter");
 }
 
+/******************************************************************************/
+/*                          d o _ S e t _ C a c h e                           */
+/******************************************************************************/
+
+// Process: set cache  
+
+int XrdXrootdProtocol::do_Set_Cache(XrdOucTokenizer &setargs)
+{
+   XrdOucErrInfo myError(Link->ID, Monitor.Did, clientPV);
+   XrdSfsFSctl myData;
+   char *cmd, *cargs, *opaque;
+   const char *myArgs[2];
+
+// This set is valid only if we implement a cache
+//
+   if ((fsFeatures & XrdSfs::hasCACH) == 0)
+      return Response.Send(kXR_ArgInvalid, "invalid set parameter");
+
+// Get the command and argument
+//
+   if (!(cmd = setargs.GetToken(&cargs)))
+      return Response.Send(kXR_ArgMissing,"set cache argument not specified.");
+
+// Prescreen the path if the next token starts with a slash
+//
+   if (cargs && *cargs == '/')
+      {if (rpCheck(cargs, &opaque)) return rpEmsg("Setting", cargs);
+       if (!Squash(cargs))          return vpEmsg("Setting", cargs);
+       myData.ArgP = myArgs; myData.Arg2Len = -2;
+       myArgs[0] = cargs;
+       myArgs[1] = opaque;
+      } else {
+       myData.Arg2 = opaque; myData.Arg2Len = (opaque ? strlen(opaque) : 0);
+      }
+   myData.Arg1 = cmd; myData.Arg1Len = strlen(cmd);
+
+// Preform the actual function using the supplied arguments
+//
+   int rc = osFS->FSctl(SFS_FSCTL_PLUGXC, myData, myError, CRED);
+   TRACEP(FS, "rc=" <prot,
                     (Client->name ? Client->name : ""),
@@ -4046,6 +4092,8 @@ void XrdXrootdProtocol::MonAuth()
                     (Entity.moninfo  ? Entity.moninfo  : ""),
                     (clientPV & XrdOucEI::uIPv4 ? '4' : '6')
                    );
+            Client->secMon = &Monitor;
+           }
 
    Monitor.Report(bP);
    if (Entity.moninfo) {free(Entity.moninfo); Entity.moninfo = 0;}
diff --git a/src/XrdZip/XrdZipCDFH.hh b/src/XrdZip/XrdZipCDFH.hh
index c7f4606cfc4..e6dbb36f23b 100644
--- a/src/XrdZip/XrdZipCDFH.hh
+++ b/src/XrdZip/XrdZipCDFH.hh
@@ -36,6 +36,8 @@
 #include 
 #include 
 
+#include 
+
 namespace XrdZip
 {
   //---------------------------------------------------------------------------
@@ -192,22 +194,22 @@ namespace XrdZip
     //-------------------------------------------------------------------------
     CDFH( const char *buffer, const uint32_t maxSize = 0 )
     {
-      zipVersion        = *reinterpret_cast( buffer + 4 );
-      minZipVersion     = *reinterpret_cast( buffer + 6 );
-      generalBitFlag    = *reinterpret_cast( buffer + 8 );
-      compressionMethod = *reinterpret_cast( buffer + 10 );
-      timestmp.time     = *reinterpret_cast( buffer + 12 );
-      timestmp.date     = *reinterpret_cast( buffer + 14 );
-      ZCRC32            = *reinterpret_cast( buffer + 16 );
-      compressedSize    = *reinterpret_cast( buffer + 20 );
-      uncompressedSize  = *reinterpret_cast( buffer + 24 );
-      filenameLength    = *reinterpret_cast( buffer + 28 );
-      extraLength       = *reinterpret_cast( buffer + 30 );
-      commentLength     = *reinterpret_cast( buffer + 32 );
-      nbDisk            = *reinterpret_cast( buffer + 34 );
-      internAttr        = *reinterpret_cast( buffer + 36 );
-      externAttr        = *reinterpret_cast( buffer + 38 );
-      offset            = *reinterpret_cast( buffer + 42 );
+      zipVersion        = to(buffer + 4);
+      minZipVersion     = to(buffer + 6);
+      generalBitFlag    = to(buffer + 8);
+      compressionMethod = to(buffer + 10);
+      timestmp.time     = to(buffer + 12);
+      timestmp.date     = to(buffer + 14);
+      ZCRC32            = to(buffer + 16);
+      compressedSize    = to(buffer + 20);
+      uncompressedSize  = to(buffer + 24);
+      filenameLength    = to(buffer + 28);
+      extraLength       = to(buffer + 30);
+      commentLength     = to(buffer + 32);
+      nbDisk            = to(buffer + 34);
+      internAttr        = to(buffer + 36);
+      externAttr        = to(buffer + 38);
+      offset            = to(buffer + 42);
       if(maxSize > 0 && (uint32_t)(cdfhBaseSize+filenameLength + extraLength + commentLength) > maxSize){
     	  throw bad_data();
       }
diff --git a/src/XrdZip/XrdZipDataDescriptor.hh b/src/XrdZip/XrdZipDataDescriptor.hh
index 22e76b5e1e0..9e48b2446fa 100644
--- a/src/XrdZip/XrdZipDataDescriptor.hh
+++ b/src/XrdZip/XrdZipDataDescriptor.hh
@@ -25,6 +25,8 @@
 #ifndef SRC_XRDZIP_XRDZIPDATADESCRIPTOR_HH_
 #define SRC_XRDZIP_XRDZIPDATADESCRIPTOR_HH_
 
+#include 
+
 namespace XrdZip
 {
   //---------------------------------------------------------------------------
diff --git a/src/XrdZip/XrdZipEOCD.hh b/src/XrdZip/XrdZipEOCD.hh
index d38a6a1c865..575a300bc0d 100644
--- a/src/XrdZip/XrdZipEOCD.hh
+++ b/src/XrdZip/XrdZipEOCD.hh
@@ -53,13 +53,13 @@ namespace XrdZip
     //-------------------------------------------------------------------------
     EOCD( const char *buffer, uint32_t maxSize = 0 )
     {
-      nbDisk        = *reinterpret_cast( buffer + 4 );
-      nbDiskCd      = *reinterpret_cast( buffer + 6 );
-      nbCdRecD      = *reinterpret_cast( buffer + 8 );
-      nbCdRec       = *reinterpret_cast( buffer + 10 );
-      cdSize        = *reinterpret_cast( buffer + 12 );
-      cdOffset      = *reinterpret_cast( buffer + 16 );
-      commentLength = *reinterpret_cast( buffer + 20 );
+      nbDisk        = to(buffer + 4);
+      nbDiskCd      = to(buffer + 6);
+      nbCdRecD      = to(buffer + 8);
+      nbCdRec       = to(buffer + 10);
+      cdSize        = to(buffer + 12);
+      cdOffset      = to(buffer + 16);
+      commentLength = to(buffer + 20);
       if(maxSize > 0 && (uint32_t)(eocdBaseSize + commentLength) > maxSize)
     	  throw bad_data();
       comment       = std::string( buffer + 22, commentLength );
diff --git a/src/XrdZip/XrdZipExtra.hh b/src/XrdZip/XrdZipExtra.hh
index c415faa7ee7..041b50ec20d 100644
--- a/src/XrdZip/XrdZipExtra.hh
+++ b/src/XrdZip/XrdZipExtra.hh
@@ -27,6 +27,9 @@
 
 #include "XrdZip/XrdZipUtils.hh"
 
+#include 
+#include 
+
 namespace XrdZip
 {
   //---------------------------------------------------------------------------
diff --git a/src/XrdZip/XrdZipLFH.hh b/src/XrdZip/XrdZipLFH.hh
index 972f373a497..aac99848eb0 100644
--- a/src/XrdZip/XrdZipLFH.hh
+++ b/src/XrdZip/XrdZipLFH.hh
@@ -80,7 +80,8 @@ namespace XrdZip
       from_buffer( minZipVersion, buffer );
       from_buffer( generalBitFlag, buffer );
       from_buffer( compressionMethod, buffer );
-      from_buffer( timestmp, buffer );
+      from_buffer( timestmp.time, buffer );
+      from_buffer( timestmp.date, buffer );
       from_buffer( ZCRC32, buffer );
       from_buffer( compressedSize, buffer );
       from_buffer( uncompressedSize, buffer );
diff --git a/src/XrdZip/XrdZipUtils.hh b/src/XrdZip/XrdZipUtils.hh
index 1d48e543d08..9b92a0a1e50 100644
--- a/src/XrdZip/XrdZipUtils.hh
+++ b/src/XrdZip/XrdZipUtils.hh
@@ -25,12 +25,14 @@
 #ifndef SRC_XRDZIP_XRDZIPUTILS_HH_
 #define SRC_XRDZIP_XRDZIPUTILS_HH_
 
-#include 
+#include "XrdSys/XrdSysPlatform.hh"
 
-#include 
-#include 
 #include 
+#include 
+#include 
+#include 
 #include 
+#include 
 
 namespace XrdZip
 {
@@ -61,7 +63,11 @@ namespace XrdZip
   {
     const char *begin = reinterpret_cast( &value );
     const char *end   = begin + sizeof( INT );
+#ifdef Xrd_Big_Endian
+    std::reverse_copy( begin, end, std::back_inserter( buffer ) );
+#else
     std::copy( begin, end, std::back_inserter( buffer ) );
+#endif
   }
 
   //---------------------------------------------------------------------------
@@ -72,6 +78,9 @@ namespace XrdZip
   inline static void from_buffer( INT &var, const char *&buffer )
   {
     memcpy( &var, buffer, sizeof( INT ) );
+#ifdef Xrd_Big_Endian
+    var = bswap(var);
+#endif
     buffer += sizeof( INT );
   }
 
@@ -82,7 +91,10 @@ namespace XrdZip
   inline static INT to( const char *buffer )
   {
     INT value;
-    memcpy( &value, buffer, sizeof( INT) );
+    memcpy( &value, buffer, sizeof( INT ) );
+#ifdef Xrd_Big_Endian
+    value = bswap(value);
+#endif
     return value;
   }
 
diff --git a/src/XrdZip/XrdZipZIP64EOCD.hh b/src/XrdZip/XrdZipZIP64EOCD.hh
index c7ab0a3bbba..09e5ecc4f39 100644
--- a/src/XrdZip/XrdZipZIP64EOCD.hh
+++ b/src/XrdZip/XrdZipZIP64EOCD.hh
@@ -28,15 +28,15 @@ namespace XrdZip
     ZIP64_EOCD( const char* buffer ):
       extensibleDataLength( 0 )
     {
-      zip64EocdSize = *reinterpret_cast( buffer + 4 );
-      zipVersion    = *reinterpret_cast( buffer + 12 );
-      minZipVersion = *reinterpret_cast( buffer + 14 );
-      nbDisk        = *reinterpret_cast( buffer + 16 );
-      nbDiskCd      = *reinterpret_cast( buffer + 20 );
-      nbCdRecD      = *reinterpret_cast( buffer + 24 );
-      nbCdRec       = *reinterpret_cast( buffer + 32 );
-      cdSize        = *reinterpret_cast( buffer + 40 );
-      cdOffset      = *reinterpret_cast( buffer + 48 );
+      zip64EocdSize = to(buffer + 4);
+      zipVersion    = to(buffer + 12);
+      minZipVersion = to(buffer + 14);
+      nbDisk        = to(buffer + 16);
+      nbDiskCd      = to(buffer + 20);
+      nbCdRecD      = to(buffer + 24);
+      nbCdRec       = to(buffer + 32);
+      cdSize        = to(buffer + 40);
+      cdOffset      = to(buffer + 48);
 
       zip64EocdTotalSize = zip64EocdBaseSize + extensibleDataLength;
     }
diff --git a/src/XrdZip/XrdZipZIP64EOCDL.hh b/src/XrdZip/XrdZipZIP64EOCDL.hh
index d2cea2edcfc..cceca756a52 100644
--- a/src/XrdZip/XrdZipZIP64EOCDL.hh
+++ b/src/XrdZip/XrdZipZIP64EOCDL.hh
@@ -26,9 +26,9 @@ namespace XrdZip
     //-------------------------------------------------------------------------
     ZIP64_EOCDL( const char *buffer )
     {
-      nbDiskZip64Eocd = *reinterpret_cast( buffer + 4 );
-      zip64EocdOffset = *reinterpret_cast( buffer + 8 );
-      totalNbDisks    = *reinterpret_cast( buffer + 16 );
+      nbDiskZip64Eocd = to(buffer + 4);
+      zip64EocdOffset = to(buffer + 8);
+      totalNbDisks    = to(buffer + 16);
     }
 
     //-------------------------------------------------------------------------
diff --git a/test.cmake b/test.cmake
new file mode 100644
index 00000000000..f86d52b10dc
--- /dev/null
+++ b/test.cmake
@@ -0,0 +1,257 @@
+cmake_minimum_required(VERSION 3.16)
+
+set(ENV{LANG} "C")
+set(ENV{LC_ALL} "C")
+set(CTEST_USE_LAUNCHERS TRUE)
+
+if(EXISTS "/etc/os-release")
+  file(STRINGS "/etc/os-release" OS_NAME REGEX "^ID=.*$")
+  string(REGEX REPLACE "ID=[\"']?([^\"']*)[\"']?$" "\\1" OS_NAME "${OS_NAME}")
+  file(STRINGS "/etc/os-release" OS_VERSION REGEX "^VERSION_ID=.*$")
+  string(REGEX REPLACE "VERSION_ID=[\"']?([^\"'.]*).*$" "\\1" OS_VERSION "${OS_VERSION}")
+  file(STRINGS "/etc/os-release" OS_FULL_NAME REGEX "^PRETTY_NAME=.*$")
+  string(REGEX REPLACE "PRETTY_NAME=[\"']?([^\"']*)[\"']?$" "\\1" OS_FULL_NAME "${OS_FULL_NAME}")
+  string(REGEX REPLACE "[ ]*\\(.*\\)" "" OS_FULL_NAME "${OS_FULL_NAME}")
+elseif(APPLE)
+  set(OS_NAME "macOS")
+  execute_process(COMMAND sw_vers -productVersion
+    OUTPUT_VARIABLE OS_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+  set(OS_FULL_NAME "${OS_NAME} ${OS_VERSION}")
+else()
+  cmake_host_system_information(RESULT OS_NAME QUERY OS_NAME)
+  cmake_host_system_information(RESULT OS_VERSION QUERY OS_VERSION)
+  set(OS_FULL_NAME "${OS_NAME} ${OS_VERSION}")
+endif()
+
+cmake_host_system_information(RESULT
+  NCORES QUERY NUMBER_OF_PHYSICAL_CORES)
+cmake_host_system_information(RESULT
+  NTHREADS QUERY NUMBER_OF_LOGICAL_CORES)
+
+if(NOT DEFINED ENV{CMAKE_BUILD_PARALLEL_LEVEL})
+  set(ENV{CMAKE_BUILD_PARALLEL_LEVEL} ${NTHREADS})
+endif()
+
+if(NOT DEFINED ENV{CTEST_PARALLEL_LEVEL})
+  set(ENV{CTEST_PARALLEL_LEVEL} ${NCORES})
+endif()
+
+if(NOT DEFINED CTEST_CONFIGURATION_TYPE)
+  if(DEFINED ENV{CMAKE_BUILD_TYPE})
+    set(CTEST_CONFIGURATION_TYPE $ENV{CMAKE_BUILD_TYPE})
+  else()
+    set(CTEST_CONFIGURATION_TYPE RelWithDebInfo)
+  endif()
+endif()
+
+execute_process(COMMAND ${CMAKE_COMMAND} --system-information
+  OUTPUT_VARIABLE CMAKE_SYSTEM_INFORMATION ERROR_VARIABLE ERROR)
+
+if(ERROR)
+  message(FATAL_ERROR "Cannot detect system information")
+endif()
+
+string(REGEX REPLACE ".+CMAKE_CXX_COMPILER_ID \"([-0-9A-Za-z ]+)\".*$" "\\1"
+  COMPILER_ID "${CMAKE_SYSTEM_INFORMATION}")
+string(REPLACE "GNU" "GCC" COMPILER_ID "${COMPILER_ID}")
+
+string(REGEX REPLACE ".+CMAKE_CXX_COMPILER_VERSION \"([^\"]+)\".*$" "\\1"
+  COMPILER_VERSION "${CMAKE_SYSTEM_INFORMATION}")
+
+set(CTEST_BUILD_NAME "${OS_FULL_NAME}")
+string(APPEND CTEST_BUILD_NAME " ${COMPILER_ID} ${COMPILER_VERSION}")
+string(APPEND CTEST_BUILD_NAME " ${CTEST_CONFIGURATION_TYPE}")
+
+if(DEFINED ENV{CMAKE_GENERATOR})
+  set(CTEST_CMAKE_GENERATOR $ENV{CMAKE_GENERATOR})
+else()
+  string(REGEX REPLACE ".+CMAKE_GENERATOR \"([-0-9A-Za-z ]+)\".*$" "\\1"
+    CTEST_CMAKE_GENERATOR "${CMAKE_SYSTEM_INFORMATION}")
+endif()
+
+if(NOT CTEST_CMAKE_GENERATOR MATCHES "Makefile")
+  string(APPEND CTEST_BUILD_NAME " ${CTEST_CMAKE_GENERATOR}")
+endif()
+
+if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
+  string(APPEND CTEST_BUILD_NAME " ${CMAKE_SYSTEM_PROCESSOR}")
+endif()
+
+if(DEFINED ENV{GITHUB_ACTIONS})
+  set(CTEST_SITE "GitHub Actions ($ENV{GITHUB_REPOSITORY_OWNER})")
+
+  if("$ENV{GITHUB_REPOSITORY_OWNER}" STREQUAL "xrootd")
+    set(CDASH TRUE)
+    set(MODEL "Continuous")
+  endif()
+
+  if("$ENV{GITHUB_EVENT_NAME}" MATCHES "pull_request")
+    set(GROUP "Pull Requests")
+    set(ENV{BASE_REF} $ENV{GITHUB_SHA}^1)
+    set(ENV{HEAD_REF} $ENV{GITHUB_SHA}^2)
+    string(REGEX REPLACE "/merge" "" PR_NUMBER "$ENV{GITHUB_REF_NAME}")
+    string(PREPEND CTEST_BUILD_NAME "#${PR_NUMBER} ($ENV{GITHUB_ACTOR})")
+  else()
+    set(ENV{HEAD_REF} $ENV{GITHUB_SHA})
+    string(APPEND CTEST_BUILD_NAME " ($ENV{GITHUB_REF_NAME})")
+  endif()
+
+  if("$ENV{GITHUB_RUN_ATTEMPT}" GREATER 1)
+    string(APPEND CTEST_BUILD_NAME " #$ENV{GITHUB_RUN_ATTEMPT}")
+  endif()
+
+  macro(section title)
+      message("::group::${title}")
+  endmacro()
+
+  macro(endsection)
+      message("::endgroup::")
+  endmacro()
+else()
+  macro(section title)
+  endmacro()
+  macro(endsection)
+  endmacro()
+endif()
+
+if(NOT DEFINED CTEST_SITE)
+  site_name(CTEST_SITE)
+endif()
+
+if(NOT DEFINED CTEST_SOURCE_DIRECTORY)
+  set(CTEST_SOURCE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}")
+endif()
+
+if(NOT DEFINED CTEST_BINARY_DIRECTORY)
+  get_filename_component(CTEST_BINARY_DIRECTORY "build" REALPATH)
+endif()
+
+if(NOT DEFINED MODEL)
+  if(DEFINED CTEST_SCRIPT_ARG)
+    set(MODEL ${CTEST_SCRIPT_ARG})
+  else()
+    set(MODEL Experimental)
+  endif()
+endif()
+
+if(NOT DEFINED GROUP)
+  set(GROUP ${MODEL})
+endif()
+
+set(CMAKE_ARGS $ENV{CMAKE_ARGS} ${CMAKE_ARGS})
+
+if(COVERAGE)
+  find_program(CTEST_COVERAGE_COMMAND NAMES gcov)
+  list(PREPEND CMAKE_ARGS "-DCMAKE_C_FLAGS=--coverage -fprofile-update=atomic")
+  list(PREPEND CMAKE_ARGS "-DCMAKE_CXX_FLAGS=--coverage -fprofile-update=atomic")
+endif()
+
+if(MEMCHECK)
+  find_program(CTEST_MEMORYCHECK_COMMAND NAMES valgrind)
+endif()
+
+if(STATIC_ANALYSIS)
+  find_program(CMAKE_CXX_CLANG_TIDY NAMES clang-tidy)
+  list(PREPEND CMAKE_ARGS "-DCMAKE_CXX_CLANG_TIDY=${CMAKE_CXX_CLANG_TIDY}")
+endif()
+
+foreach(FILENAME ${OS_NAME}${OS_VERSION}.cmake ${OS_NAME}.cmake config.cmake)
+  if(EXISTS "${CTEST_SOURCE_DIRECTORY}/.ci/${FILENAME}")
+    message(STATUS "Using CMake cache file ${FILENAME}")
+    list(PREPEND CMAKE_ARGS -C ${CTEST_SOURCE_DIRECTORY}/.ci/${FILENAME})
+    list(APPEND CTEST_NOTES_FILES ${CTEST_SOURCE_DIRECTORY}/.ci/${FILENAME})
+    break()
+  endif()
+endforeach()
+
+if(IS_DIRECTORY "${CTEST_BINARY_DIRECTORY}")
+  ctest_empty_binary_directory("${CTEST_BINARY_DIRECTORY}")
+endif()
+
+ctest_read_custom_files("${CTEST_SOURCE_DIRECTORY}")
+
+if(IS_DIRECTORY ${CTEST_SOURCE_DIRECTORY}/.git)
+  find_program(CTEST_GIT_COMMAND NAMES git)
+endif()
+
+if(EXISTS "${CTEST_GIT_COMMAND}" AND DEFINED ENV{HEAD_REF} AND NOT DEFINED ENV{BASE_REF})
+  set(CTEST_CHECKOUT_COMMAND
+    "${CTEST_GIT_COMMAND} --git-dir ${CTEST_SOURCE_DIRECTORY}/.git checkout -f $ENV{HEAD_REF}")
+endif()
+
+ctest_start(${MODEL} GROUP "${GROUP}")
+
+if(EXISTS "${CTEST_GIT_COMMAND}")
+  if(DEFINED ENV{BASE_REF})
+    execute_process(COMMAND ${CTEST_GIT_COMMAND} checkout -f $ENV{BASE_REF}
+      WORKING_DIRECTORY ${CTEST_SOURCE_DIRECTORY} ERROR_QUIET RESULT_VARIABLE GIT_STATUS)
+    if(NOT ${GIT_STATUS} EQUAL 0)
+      message(FATAL_ERROR "Could not checkout base ref: $ENV{BASE_REF}")
+    endif()
+  endif()
+  if(DEFINED ENV{HEAD_REF})
+    set(CTEST_GIT_UPDATE_CUSTOM
+      ${CTEST_GIT_COMMAND} --git-dir ${CTEST_SOURCE_DIRECTORY}/.git checkout -f $ENV{HEAD_REF})
+  else()
+    set(CTEST_GIT_UPDATE_CUSTOM ${CTEST_GIT_COMMAND} --git-dir ${CTEST_SOURCE_DIRECTORY}/.git diff)
+  endif()
+
+  ctest_update()
+endif()
+
+section("Configure")
+ctest_configure(OPTIONS "${CMAKE_ARGS}")
+
+ctest_read_custom_files("${CTEST_BINARY_DIRECTORY}")
+list(APPEND CTEST_NOTES_FILES ${CTEST_BINARY_DIRECTORY}/CMakeCache.txt)
+endsection()
+
+section("Build")
+ctest_build(RETURN_VALUE BUILD_RESULT)
+
+if(NOT ${BUILD_RESULT} EQUAL 0)
+  if(CDASH OR (DEFINED ENV{CDASH} AND "$ENV{CDASH}"))
+    ctest_submit()
+  endif()
+  message(FATAL_ERROR "Build failed")
+endif()
+
+if(INSTALL)
+  set(ENV{DESTDIR} "${CTEST_BINARY_DIRECTORY}/install")
+  ctest_build(TARGET install)
+endif()
+endsection()
+
+section("Test")
+ctest_test(PARALLEL_LEVEL $ENV{CTEST_PARALLEL_LEVEL} RETURN_VALUE TEST_RESULT)
+
+if(NOT ${TEST_RESULT} EQUAL 0)
+  message(SEND_ERROR "Tests failed")
+endif()
+endsection()
+
+if(DEFINED CTEST_COVERAGE_COMMAND)
+  section("Coverage")
+  find_program(GCOVR NAMES gcovr)
+  if(EXISTS ${GCOVR})
+    execute_process(COMMAND
+      ${GCOVR} --gcov-executable ${CTEST_COVERAGE_COMMAND}
+        -r ${CTEST_SOURCE_DIRECTORY} ${CTEST_BINARY_DIRECTORY}
+        --html-details ${CTEST_BINARY_DIRECTORY}/coverage/ ERROR_VARIABLE ERROR)
+    if(ERROR)
+      message(SEND_ERROR "Failed to generate coverage report")
+    endif()
+  endif()
+  ctest_coverage()
+  endsection()
+endif()
+
+if(DEFINED CTEST_MEMORYCHECK_COMMAND)
+  section("Memcheck")
+  ctest_memcheck()
+  endsection()
+endif()
+
+if(CDASH OR (DEFINED ENV{CDASH} AND "$ENV{CDASH}"))
+  ctest_submit()
+endif()
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index d26a902851f..070d7bf7dfd 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1,3 +1,7 @@
+if(NOT BUILD_TESTS)
+  return()
+endif()
+
 include(GoogleTest)
 add_subdirectory( XrdCl )
 add_subdirectory(XrdHttpTests)
@@ -8,8 +12,12 @@ add_subdirectory( XrdSsiTests )
 
 if( BUILD_XRDEC )
   add_subdirectory( XrdEcTests )
+  add_subdirectory( XrdEc ) # new tests with GTest
 endif()
 
 if( BUILD_CEPH )
   add_subdirectory( XrdCephTests )
 endif()
+
+add_subdirectory( XRootD )
+add_subdirectory( cluster )
diff --git a/tests/XRootD/CMakeLists.txt b/tests/XRootD/CMakeLists.txt
new file mode 100644
index 00000000000..7ac633ba807
--- /dev/null
+++ b/tests/XRootD/CMakeLists.txt
@@ -0,0 +1,34 @@
+if(XRDCL_ONLY)
+  return()
+endif()
+
+execute_process(COMMAND id -u OUTPUT_VARIABLE UID OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if (UID EQUAL 0)
+  return()
+endif()
+
+set(XRD_TEST_PORT "11940" CACHE STRING "Port for XRootD Test Server")
+
+list(APPEND XRDENV "XRDCP=$")
+list(APPEND XRDENV "XRDFS=$")
+list(APPEND XRDENV "CRC32C=$")
+list(APPEND XRDENV "ADLER32=$")
+list(APPEND XRDENV "HOST=root://localhost:${XRD_TEST_PORT}")
+
+configure_file(xrootd.cfg xrootd.cfg @ONLY)
+
+add_test(NAME XRootD::start
+  COMMAND sh -c "mkdir -p data && \
+  $ -b -k fifo -n standalone -l xrootd.log -s xrootd.pid -c xrootd.cfg")
+set_tests_properties(XRootD::start PROPERTIES FIXTURES_SETUP   XRootD)
+
+add_test(NAME XRootD::stop
+  COMMAND sh -c "kill -s TERM $(cat standalone/xrootd.pid); rm -rf data standalone")
+set_tests_properties(XRootD::stop  PROPERTIES FIXTURES_CLEANUP XRootD)
+
+add_test(NAME XRootD::smoke-test
+  COMMAND sh -c "${CMAKE_CURRENT_SOURCE_DIR}/smoke.sh")
+
+set_tests_properties(XRootD::smoke-test PROPERTIES
+  ENVIRONMENT "${XRDENV}" FIXTURES_REQUIRED XRootD)
diff --git a/tests/XRootD/smoke.sh b/tests/XRootD/smoke.sh
new file mode 100755
index 00000000000..76ae67fd9ab
--- /dev/null
+++ b/tests/XRootD/smoke.sh
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+
+set -e
+
+: ${ADLER32:=$(command -v xrdadler32)}
+: ${CRC32C:=$(command -v xrdcrc32c)}
+: ${XRDCP:=$(command -v xrdcp)}
+: ${XRDFS:=$(command -v xrdfs)}
+: ${OPENSSL:=$(command -v openssl)}
+: ${HOST:=root://localhost:${PORT:-1094}}
+
+for PROG in ${ADLER32} ${CRC32C} ${XRDCP} ${XRDFS} ${OPENSSL}; do
+       if [[ ! -x "${PROG}" ]]; then
+               echo 1>&2 "$(basename $0): error: '${PROG}': command not found"
+               exit 1
+       fi
+done
+
+# This script assumes that ${HOST} exports an empty / as read/write.
+# It also assumes that any authentication required is already setup.
+
+echo Using ${OPENSSL}: $(${OPENSSL} version)
+echo Using ${XRDCP}: $(${XRDCP} --version)
+
+${XRDFS} ${HOST} query config version
+
+# query some common server configurations
+
+CONFIG_PARAMS=( version role sitename )
+
+for PARAM in ${CONFIG_PARAMS[@]}; do
+       ${XRDFS} ${HOST} query config ${PARAM}
+done
+
+# some extra query commands that don't make any changes
+
+${XRDFS} ${HOST} stat /
+${XRDFS} ${HOST} statvfs /
+${XRDFS} ${HOST} spaceinfo /
+
+# create local temporary directory
+TMPDIR=$(mktemp -d ${PWD}/xrdfs-test-XXXXXX)
+
+# cleanup after ourselves if something fails
+trap "rm -rf ${TMPDIR}" EXIT
+
+# create remote temporary directory
+# this will get cleaned up by CMake upon fixture tear down
+${XRDFS} ${HOST} mkdir -p ${TMPDIR}
+
+# create local files with random contents using OpenSSL
+
+FILES=$(seq -w 1 ${NFILES:-10})
+
+for i in $FILES; do
+       ${OPENSSL} rand -out "${TMPDIR}/${i}.ref" $((1024 * $RANDOM))
+done
+
+# upload local files to the server in parallel
+
+for i in $FILES; do
+       ${XRDCP} ${TMPDIR}/${i}.ref ${HOST}/${TMPDIR}/${i}.ref
+done
+
+# list uploaded files, then download them to check for corruption
+
+${XRDFS} ${HOST} ls -l ${TMPDIR}
+
+for i in $FILES; do
+       ${XRDCP} ${HOST}/${TMPDIR}/${i}.ref ${TMPDIR}/${i}.dat
+done
+
+# check that all checksums for downloaded files match
+
+for i in $FILES; do
+    REF32C=$(${CRC32C} < ${TMPDIR}/${i}.ref | cut -d' '  -f1)
+    NEW32C=$(${CRC32C} < ${TMPDIR}/${i}.dat | cut -d' '  -f1)
+
+    REFA32=$(${ADLER32} < ${TMPDIR}/${i}.ref | cut -d' '  -f1)
+    NEWA32=$(${ADLER32} < ${TMPDIR}/${i}.dat | cut -d' '  -f1)
+
+    if setfattr -n user.checksum -v ${REF32C} ${TMPDIR}/${i}.ref; then
+        SRV32C=$(${XRDFS} ${HOST} query checksum ${TMPDIR}/${i}.ref?cks.type=crc32c | cut -d' ' -f2)
+        SRVA32=$(${XRDFS} ${HOST} query checksum ${TMPDIR}/${i}.ref?cks.type=adler32 | cut -d' ' -f2)
+    else
+        echo "Extended attributes not supported, using downloaded checksums for server checks"
+        SRV32C=${NEW32C} SRVA32=${NEWA32} # use downloaded file checksum if xattr not supported
+    fi
+
+    if [[ "${NEWA32}" != "${REFA32}" || "${SRVA32}" != "${REFA32}" ]]; then
+        echo 1>&2 "$(basename $0): error: adler32 checksum check failed for file: ${i}.dat"
+        echo 1>&2 "${i}: adler32: reference: ${REFA32}, server: ${SRVA32}, downloaded: ${NEWA32}"
+        exit 1
+    fi
+
+    if [[ "${NEW32C}" != "${REF32C}" || "${SRV32C}" != "${REF32C}" ]]; then
+        echo 1>&2 "$(basename $0): error: crc32 checksum check failed for file: ${i}.dat"
+        echo 1>&2 "${i}:  crc32c: reference: ${REF32C}, server: ${SRV32C}, downloaded: ${NEW32C}"
+        exit 1
+    fi
+done
+
+for i in $FILES; do
+       ${XRDFS} ${HOST} rm ${TMPDIR}/${i}.ref &
+done
+
+wait
+
+#
+# check return code for xrdfs rm
+# create another 6 files, which should be deleted during the test
+#
+for i in $(seq -w 1 6) ; do
+       ${OPENSSL} rand -out "${TMPDIR}/${i}.exists.ref" $((1024 * $RANDOM))
+       ${XRDCP} ${TMPDIR}/${i}.exists.ref ${HOST}/${TMPDIR}/${i}.exists.ref
+done
+
+# remove 3 existing, should succeed not error
+${XRDFS} ${HOST} rm ${TMPDIR}/1.exists.ref ${TMPDIR}/2.exists.ref ${TMPDIR}/3.exists.ref
+
+set +e
+
+# remove 3 not existing, should error
+#
+${XRDFS} ${HOST} rm ${TMPDIR}/not_exists_1.ref ${TMPDIR}/not_exists_2.ref ${TMPDIR}/not_exists_3.ref
+rm_rc=$?
+if [ $rm_rc -eq 0 ]; then
+  exit 1
+fi
+#
+# remove 2 existing, 1 not existing should error
+#
+${XRDFS} ${HOST} rm ${TMPDIR}/4.exists.ref ${TMPDIR}/5.exists.ref ${TMPDIR}/not_exists_4.ref
+rm_rc=$?
+if [ $rm_rc -eq 0 ]; then
+  exit 1
+fi
+#
+# remove 1 existing, 2 not existing should error
+#
+${XRDFS} ${HOST} rm ${TMPDIR}/6.exists.ref ${TMPDIR}/not_exists_5.ref ${TMPDIR}/not_exists_6.ref
+rm_rc=$?
+if [ $rm_rc -eq 0 ]; then
+  exit 1
+fi
+set -e
+
+${XRDFS} ${HOST} rmdir ${TMPDIR}
+
+echo "ALL TESTS PASSED"
+exit 0
diff --git a/tests/XRootD/xrootd.cfg b/tests/XRootD/xrootd.cfg
new file mode 100644
index 00000000000..7351b1e3436
--- /dev/null
+++ b/tests/XRootD/xrootd.cfg
@@ -0,0 +1,10 @@
+# This minimal configuration file starts a standalone server
+# that exports the data directory as / without authentication.
+
+all.export /
+all.sitename XRootD
+all.adminpath @CMAKE_CURRENT_BINARY_DIR@
+all.pidpath   @CMAKE_CURRENT_BINARY_DIR@
+oss.localroot @CMAKE_CURRENT_BINARY_DIR@/data
+xrd.port @XRD_TEST_PORT@
+xrootd.chksum chkcgi adler32 crc32c
diff --git a/tests/XrdCephTests/CMakeLists.txt b/tests/XrdCephTests/CMakeLists.txt
index 383a6fa70f5..ed05bc465d0 100644
--- a/tests/XrdCephTests/CMakeLists.txt
+++ b/tests/XrdCephTests/CMakeLists.txt
@@ -1,6 +1,3 @@
-include( XRootDCommon )
-include_directories( ${CPPUNIT_INCLUDE_DIRS} )
-
 add_library(
   XrdCephTests MODULE
   CephParsingTest.cc
@@ -13,6 +10,8 @@ target_link_libraries(
   ZLIB::ZLIB
   XrdCephPosix )
 
+target_include_directories( XrdCephTests PRIVATE ${CPPUNIT_INCLUDE_DIRS} )
+
 #-------------------------------------------------------------------------------
 # Install
 #-------------------------------------------------------------------------------
diff --git a/tests/XrdCephTests/CephParsingTest.cc b/tests/XrdCephTests/CephParsingTest.cc
index 194bd5e40c8..06cb3366097 100644
--- a/tests/XrdCephTests/CephParsingTest.cc
+++ b/tests/XrdCephTests/CephParsingTest.cc
@@ -69,12 +69,12 @@ void checkResult(CephFile a, CephFile b) {
             << " / " << b.name << " " << b.pool << " " << b.userId
             << " " << b.nbStripes << " " << b.stripeUnit << " " << b.objectSize
             << std::endl;
-  CPPUNIT_ASSERT(a.name == b.name);
-  CPPUNIT_ASSERT(a.pool == b.pool);
-  CPPUNIT_ASSERT(a.userId == b.userId);
-  CPPUNIT_ASSERT(a.nbStripes == b.nbStripes);
-  CPPUNIT_ASSERT(a.stripeUnit == b.stripeUnit);
-  CPPUNIT_ASSERT(a.objectSize == b.objectSize);
+  CPPUNIT_ASSERT_EQUAL(a.name, b.name);
+  CPPUNIT_ASSERT_EQUAL(a.pool, b.pool);
+  CPPUNIT_ASSERT_EQUAL(a.userId, b.userId);
+  CPPUNIT_ASSERT_EQUAL(a.nbStripes, b.nbStripes);
+  CPPUNIT_ASSERT_EQUAL(a.stripeUnit, b.stripeUnit);
+  CPPUNIT_ASSERT_EQUAL(a.objectSize, b.objectSize);
 }
 
 //------------------------------------------------------------------------------
diff --git a/tests/XrdCl/CMakeLists.txt b/tests/XrdCl/CMakeLists.txt
index a114c4cb620..015f4c92501 100644
--- a/tests/XrdCl/CMakeLists.txt
+++ b/tests/XrdCl/CMakeLists.txt
@@ -1,18 +1,103 @@
-
 add_executable(xrdcl-unit-tests
   XrdClURL.cc
-)
+  XrdClPoller.cc
+  XrdClSocket.cc
+  XrdClUtilsTest.cc
+  ../common/Server.cc
+  ../common/Utils.cc
+  ../common/TestEnv.cc
+  )
 
 target_link_libraries(xrdcl-unit-tests
   XrdCl
   XrdXml
   XrdUtils
+  ZLIB::ZLIB
   GTest::GTest
   GTest::Main
 )
 
 target_include_directories(xrdcl-unit-tests
-  PRIVATE ${CMAKE_SOURCE_DIR}/src
+  PRIVATE ${CMAKE_SOURCE_DIR}/src ../common
 )
 
 gtest_discover_tests(xrdcl-unit-tests TEST_PREFIX XrdCl::)
+
+if(XRDCL_ONLY)
+  return()
+endif()
+
+execute_process(COMMAND id -u OUTPUT_VARIABLE UID OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if (UID EQUAL 0)
+  return()
+endif()
+
+file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tmp")
+
+add_executable(xrdcl-cluster-tests
+  IdentityPlugIn.cc
+  # XrdClFileTest.cc
+  # XrdClFileCopyTest.cc
+  # XrdClFileSystemTest.cc
+  # XrdClOperationsWorkflowTest.cc
+  XrdClLocalFileHandlerTest.cc
+  XrdClPostMasterTest.cc
+  XrdClThreadingTest.cc
+  XrdClZip.cc
+  ../common/Server.cc
+  ../common/Utils.cc
+  ../common/TestEnv.cc
+  )
+
+target_link_libraries(xrdcl-cluster-tests
+  XrdCl
+  XrdXml
+  XrdUtils
+  ZLIB::ZLIB
+  GTest::GTest
+  GTest::Main
+)
+
+target_include_directories(xrdcl-cluster-tests
+  PRIVATE ${CMAKE_SOURCE_DIR}/src ../common
+)
+
+gtest_discover_tests(xrdcl-cluster-tests TEST_PREFIX XrdCl::
+  PROPERTIES DEPENDS XRootD_Cluster FIXTURES_REQUIRED XRootD_Cluster)
+
+# tests to be run in serial, otherwise they are problematic
+set(SERIAL_TESTS_FILES
+  XrdClFileTest.cc
+  XrdClFileCopyTest.cc
+  XrdClFileSystemTest.cc
+  XrdClOperationsWorkflowTest.cc
+)
+
+# create a separate executable target for each "problematic" test suite
+foreach(i ${SERIAL_TESTS_FILES})
+  add_executable(${i}-tests
+    ${i}
+    IdentityPlugIn.cc
+    ../common/Server.cc
+    ../common/Utils.cc
+    ../common/TestEnv.cc
+  )
+
+  target_link_libraries(${i}-tests
+    XrdCl
+    XrdXml
+    XrdUtils
+    ZLIB::ZLIB
+    GTest::GTest
+    GTest::Main
+  )
+
+  target_include_directories(${i}-tests
+    PRIVATE ${CMAKE_SOURCE_DIR}/src ../common
+  )
+
+  gtest_discover_tests(${i}-tests TEST_PREFIX XrdCl::
+    PROPERTIES DEPENDS XRootD_Cluster FIXTURES_REQUIRED XRootD_Cluster RUN_SERIAL 1)
+
+endforeach()
diff --git a/tests/XrdCl/GTestXrdHelpers.hh b/tests/XrdCl/GTestXrdHelpers.hh
new file mode 100644
index 00000000000..d54ef29c95a
--- /dev/null
+++ b/tests/XrdCl/GTestXrdHelpers.hh
@@ -0,0 +1,70 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+#ifndef __GTEST_XRD_HELPERS_HH__
+#define __GTEST_XRD_HELPERS_HH__
+
+#include 
+#include 
+#include 
+#include 
+
+/** @brief Equivalent of CPPUNIT_ASSERT_XRDST
+ *
+ * Shows the code that we are asserting and its value
+ * in the final evaluation.
+ */
+#define GTEST_ASSERT_XRDST( x )                                                                  \
+{                                                                                                \
+  XrdCl::XRootDStatus _st = x;                                                                   \
+  EXPECT_TRUE(_st.IsOK()) << "[" << #x << "]: " << _st.ToStr() << std::endl;                     \
+}
+
+/** @brief Equivalent of CPPUNIT_ASSERT_XRDST_NOTOK
+ *
+ * Shows the code that we are asserting and asserts that its
+ * execution is throwing an error.
+ */
+#define GTEST_ASSERT_XRDST_NOTOK( x, err )                                                       \
+{                                                                                                \
+  XrdCl::XRootDStatus _st = x;                                                                   \
+  EXPECT_TRUE(!_st.IsOK() && _st.code == err) << "[" << #x << "]: " << _st.ToStr() << std::endl; \
+}
+
+/** @brief Equivalent of CPPUNIT_ASSERT_ERRNO
+ *
+ * Shows the code that we are asserting and its error
+ * number.
+ */
+#define GTEST_ASSERT_ERRNO( x )                                                                  \
+{                                                                                                \
+  EXPECT_TRUE(x) << "[" << #x << "]: " << strerror(errno) << std::endl;                          \
+}
+
+/** @brief Equivalent of GTEST_ASSERT_PTHREAD
+ *
+ * Shows the code that we are asserting and its error
+ * number, in a thread-safe manner.
+ */
+#define GTEST_ASSERT_PTHREAD( x )                                                                \
+{                                                                                                \
+  errno = x;                                                                                     \
+  EXPECT_TRUE(errno == 0) << "[" << #x << "]: " << strerror(errno) << std::endl;                 \
+}
+
+#endif // __GTEST_XRD_HELPERS_HH__
diff --git a/tests/XrdCl/IdentityPlugIn.cc b/tests/XrdCl/IdentityPlugIn.cc
new file mode 100644
index 00000000000..eea454e50c0
--- /dev/null
+++ b/tests/XrdCl/IdentityPlugIn.cc
@@ -0,0 +1,488 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// This file is part of the XRootD software suite.
+//
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//
+// In applying this licence, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+//------------------------------------------------------------------------------
+
+#include "XrdCl/XrdClFile.hh"
+#include "XrdCl/XrdClFileSystem.hh"
+#include "XrdCl/XrdClPlugInInterface.hh"
+#include "XrdCl/XrdClLog.hh"
+#include "IdentityPlugIn.hh"
+#include "TestEnv.hh"
+
+using namespace XrdCl;
+using namespace XrdClTests;
+
+namespace
+{
+  //----------------------------------------------------------------------------
+  // A plugin that forwards all the calls to XrdCl::File
+  //----------------------------------------------------------------------------
+  class IdentityFile: public XrdCl::FilePlugIn
+  {
+    public:
+      //------------------------------------------------------------------------
+      // Constructor
+      //------------------------------------------------------------------------
+      IdentityFile()
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::IdentityFile" );
+        pFile = new File( false );
+      }
+
+      //------------------------------------------------------------------------
+      // Destructor
+      //------------------------------------------------------------------------
+      virtual ~IdentityFile()
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::~IdentityFile" );
+        delete pFile;
+      }
+
+      //------------------------------------------------------------------------
+      // Open
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Open( const std::string &url,
+                                 OpenFlags::Flags   flags,
+                                 Access::Mode       mode,
+                                 ResponseHandler   *handler,
+                                 uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::Open" );
+        return pFile->Open( url, flags, mode, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Close
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Close( ResponseHandler *handler,
+                                  uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::Close" );
+        return pFile->Close( handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Stat
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Stat( bool             force,
+                                 ResponseHandler *handler,
+                                 uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::Stat" );
+        return pFile->Stat( force, handler, timeout );
+      }
+
+
+      //------------------------------------------------------------------------
+      // Read
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Read( uint64_t         offset,
+                                 uint32_t         size,
+                                 void            *buffer,
+                                 ResponseHandler *handler,
+                                 uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::Read" );
+        return pFile->Read( offset, size, buffer, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Write
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Write( uint64_t         offset,
+                                  uint32_t         size,
+                                  const void      *buffer,
+                                  ResponseHandler *handler,
+                                  uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::Write" );
+        return pFile->Write( offset, size, buffer, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Sync
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Sync( ResponseHandler *handler,
+                                 uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::Sync" );
+        return pFile->Sync( handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Truncate
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Truncate( uint64_t         size,
+                                     ResponseHandler *handler,
+                                     uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::Truncate" );
+        return pFile->Truncate( size, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // VectorRead
+      //------------------------------------------------------------------------
+      virtual XRootDStatus VectorRead( const ChunkList &chunks,
+                                       void            *buffer,
+                                       ResponseHandler *handler,
+                                       uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::VectorRead" );
+        return pFile->VectorRead( chunks, buffer, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Fcntl
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Fcntl( const Buffer    &arg,
+                                  ResponseHandler *handler,
+                                  uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::Fcntl" );
+        return pFile->Fcntl( arg, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Visa
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Visa( ResponseHandler *handler,
+                                 uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::Visa" );
+        return pFile->Visa( handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // IsOpen
+      //------------------------------------------------------------------------
+      virtual bool IsOpen() const
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::IsOpen" );
+        return pFile->IsOpen();
+      }
+
+      //------------------------------------------------------------------------
+      // SetProperty
+      //------------------------------------------------------------------------
+      virtual bool SetProperty( const std::string &name,
+                                const std::string &value )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::SetProperty" );
+        return pFile->SetProperty( name, value );
+      }
+
+      //------------------------------------------------------------------------
+      // GetProperty
+      //------------------------------------------------------------------------
+      virtual bool GetProperty( const std::string &name,
+                                std::string &value ) const
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFile::GetProperty" );
+        return pFile->GetProperty( name, value );
+      }
+
+    private:
+      XrdCl::File *pFile;
+  };
+
+  //----------------------------------------------------------------------------
+  // A plug-in that forwards all the calls to a XrdCl::FileSystem object
+  //----------------------------------------------------------------------------
+  class IdentityFileSystem: public FileSystemPlugIn
+  {
+    public:
+      //------------------------------------------------------------------------
+      // Constructor
+      //------------------------------------------------------------------------
+      IdentityFileSystem( const std::string &url )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::IdentityFileSystem" );
+        pFileSystem = new XrdCl::FileSystem( URL(url), false );
+      }
+
+      //------------------------------------------------------------------------
+      // Destructor
+      //------------------------------------------------------------------------
+      virtual ~IdentityFileSystem()
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::~IdentityFileSysytem" );
+        delete pFileSystem;
+      }
+
+      //------------------------------------------------------------------------
+      // Locate
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Locate( const std::string &path,
+                                   OpenFlags::Flags   flags,
+                                   ResponseHandler   *handler,
+                                   uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::Locate" );
+        return pFileSystem->Locate( path, flags, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Mv
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Mv( const std::string &source,
+                               const std::string &dest,
+                               ResponseHandler   *handler,
+                               uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::Mv" );
+        return pFileSystem->Mv( source, dest, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Query
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Query( QueryCode::Code  queryCode,
+                                  const Buffer    &arg,
+                                  ResponseHandler *handler,
+                                  uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::Query" );
+        return pFileSystem->Query( queryCode, arg, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Truncate
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Truncate( const std::string &path,
+                                     uint64_t           size,
+                                     ResponseHandler   *handler,
+                                     uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::Truncate" );
+        return pFileSystem->Truncate( path, size, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Rm
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Rm( const std::string &path,
+                               ResponseHandler   *handler,
+                               uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::Rm" );
+        return pFileSystem->Rm( path, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // MkDir
+      //------------------------------------------------------------------------
+      virtual XRootDStatus MkDir( const std::string &path,
+                                  MkDirFlags::Flags  flags,
+                                  Access::Mode       mode,
+                                  ResponseHandler   *handler,
+                                  uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::MkDir" );
+        return pFileSystem->MkDir( path, flags, mode, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // RmDir
+      //------------------------------------------------------------------------
+      virtual XRootDStatus RmDir( const std::string &path,
+                                  ResponseHandler   *handler,
+                                  uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::RmDir" );
+        return pFileSystem->RmDir( path, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // ChMod
+      //------------------------------------------------------------------------
+      virtual XRootDStatus ChMod( const std::string &path,
+                                  Access::Mode       mode,
+                                  ResponseHandler   *handler,
+                                  uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::ChMod" );
+        return pFileSystem->ChMod( path, mode, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Ping
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Ping( ResponseHandler *handler,
+                                 uint16_t         timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::Ping" );
+        return pFileSystem->Ping( handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Stat
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Stat( const std::string &path,
+                                 ResponseHandler   *handler,
+                                 uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::Stat" );
+        return pFileSystem->Stat( path, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // StatVFS
+      //------------------------------------------------------------------------
+      virtual XRootDStatus StatVFS( const std::string &path,
+                                    ResponseHandler   *handler,
+                                    uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::StatVFS" );
+        return pFileSystem->StatVFS( path, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Protocol
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Protocol( ResponseHandler *handler,
+                                     uint16_t         timeout = 0 )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::Protocol" );
+        return pFileSystem->Protocol( handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // DirlList
+      //------------------------------------------------------------------------
+      virtual XRootDStatus DirList( const std::string   &path,
+                                    DirListFlags::Flags  flags,
+                                    ResponseHandler     *handler,
+                                    uint16_t             timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::DirList" );
+        return pFileSystem->DirList( path, flags, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // SendInfo
+      //------------------------------------------------------------------------
+      virtual XRootDStatus SendInfo( const std::string &info,
+                                     ResponseHandler   *handler,
+                                     uint16_t           timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::SendInfo" );
+        return pFileSystem->SendInfo( info, handler, timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // Prepare
+      //------------------------------------------------------------------------
+      virtual XRootDStatus Prepare( const std::vector &fileList,
+                                    PrepareFlags::Flags             flags,
+                                    uint8_t                         priority,
+                                    ResponseHandler                *handler,
+                                    uint16_t                        timeout )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::Prepare" );
+        return pFileSystem->Prepare( fileList, flags, priority, handler,
+                                     timeout );
+      }
+
+      //------------------------------------------------------------------------
+      // SetProperty
+      //------------------------------------------------------------------------
+      virtual bool SetProperty( const std::string &name,
+                                const std::string &value )
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFileSystem::SetProperty" );
+        return pFileSystem->SetProperty( name, value );
+      }
+
+      //------------------------------------------------------------------------
+      // GetProperty
+      //------------------------------------------------------------------------
+      virtual bool GetProperty( const std::string &name,
+                                std::string &value ) const
+      {
+        XrdCl::Log *log = TestEnv::GetLog();
+        log->Debug( 1, "Calling IdentityFilesystem::GetProperty" );
+        return pFileSystem->GetProperty( name, value );
+      }
+
+    private:
+      XrdCl::FileSystem *pFileSystem;
+  };
+}
+
+namespace XrdClTests
+{
+  //----------------------------------------------------------------------------
+  // Create a file plug-in for the given URL
+  //----------------------------------------------------------------------------
+  FilePlugIn *IdentityFactory::CreateFile( const std::string &url )
+  {
+    XrdCl::Log *log = TestEnv::GetLog();
+    log->Debug( 1, "Creating an identity file plug-in" );
+    return new IdentityFile();
+  }
+
+  //----------------------------------------------------------------------------
+  // Create a file system plug-in for the given URL
+  //----------------------------------------------------------------------------
+  FileSystemPlugIn *IdentityFactory::CreateFileSystem( const std::string &url )
+  {
+    XrdCl::Log *log = TestEnv::GetLog();
+    log->Debug( 1, "Creating an identity file system plug-in" );
+    return new IdentityFileSystem( url );
+  }
+}
+
diff --git a/tests/XrdCl/IdentityPlugIn.hh b/tests/XrdCl/IdentityPlugIn.hh
new file mode 100644
index 00000000000..7f6fb0d5c90
--- /dev/null
+++ b/tests/XrdCl/IdentityPlugIn.hh
@@ -0,0 +1,55 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// This file is part of the XRootD software suite.
+//
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//
+// In applying this licence, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+//------------------------------------------------------------------------------
+
+#ifndef __XRDCLTESTS_IDENTITY_PLUGIN_HH__
+#define __XRDCLTESTS_IDENTITY_PLUGIN_HH__
+
+#include "XrdCl/XrdClPlugInInterface.hh"
+
+namespace XrdClTests
+{
+  //----------------------------------------------------------------------------
+  // Plugin factory
+  //----------------------------------------------------------------------------
+  class IdentityFactory: public XrdCl::PlugInFactory
+  {
+    public:
+      //------------------------------------------------------------------------
+      // Destructor
+      //------------------------------------------------------------------------
+      virtual ~IdentityFactory() {}
+
+      //------------------------------------------------------------------------
+      // Create a file plug-in for the given URL
+      //------------------------------------------------------------------------
+      virtual XrdCl::FilePlugIn *CreateFile( const std::string &url );
+
+      //------------------------------------------------------------------------
+      // Create a file system plug-in for the given URL
+      //------------------------------------------------------------------------
+      virtual XrdCl::FileSystemPlugIn *CreateFileSystem( const std::string &url );
+  };
+};
+
+#endif // __XRDCLTESTS_IDENTITY_PLUGIN_HH__
diff --git a/tests/XrdCl/XrdClFile.cc b/tests/XrdCl/XrdClFile.cc
new file mode 100644
index 00000000000..d98817ffd2a
--- /dev/null
+++ b/tests/XrdCl/XrdClFile.cc
@@ -0,0 +1,38 @@
+#undef NDEBUG
+
+#include 
+#include 
+
+#include 
+
+using namespace testing;
+
+class FileTest : public ::testing::Test {};
+
+TEST(FileTest, StreamTimeout)
+{
+  XrdCl::Env *env  = XrdCl::DefaultEnv::GetEnv();
+
+  env->PutInt("StreamTimeout", 1); //60 is default
+  env->PutInt("TimeoutResolution", 0); //15 is default
+
+  char buf[16];
+  uint32_t BytesRead = 0;
+  XrdCl::File f;
+
+  f.SetProperty("ReadRecovery", "false");
+
+  auto st = f.Open("root://localhost//test.txt", XrdCl::OpenFlags::Read);
+
+  EXPECT_TRUE(st.IsOK()) << "Open not OK:" << st.ToString() << std::endl;
+
+  sleep(3); // wait for timeout
+
+  st = f.Read(0, 5, buf, BytesRead, 0);
+
+  EXPECT_TRUE(st.IsOK()) << "Read not OK:" << st.ToString() << std::endl;
+
+  st = f.Close();
+
+  EXPECT_TRUE(st.IsOK()) << "Close not OK:" << st.ToString() << std::endl;
+}
diff --git a/tests/XrdCl/XrdClFileCopyTest.cc b/tests/XrdCl/XrdClFileCopyTest.cc
new file mode 100644
index 00000000000..d991b358e15
--- /dev/null
+++ b/tests/XrdCl/XrdClFileCopyTest.cc
@@ -0,0 +1,656 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+#include "TestEnv.hh"
+#include "GTestXrdHelpers.hh"
+#include "XrdCl/XrdClFile.hh"
+#include "XrdCl/XrdClDefaultEnv.hh"
+#include "XrdCl/XrdClMessage.hh"
+#include "XrdCl/XrdClSIDManager.hh"
+#include "XrdCl/XrdClPostMaster.hh"
+#include "XrdCl/XrdClXRootDTransport.hh"
+#include "XrdCl/XrdClMessageUtils.hh"
+#include "XrdCl/XrdClXRootDMsgHandler.hh"
+#include "XrdCl/XrdClUtils.hh"
+#include "XrdCl/XrdClCheckSumManager.hh"
+#include "XrdCl/XrdClCopyProcess.hh"
+
+#include "XrdCks/XrdCks.hh"
+#include "XrdCks/XrdCksCalc.hh"
+#include "XrdCks/XrdCksData.hh"
+
+#include 
+#include 
+#include 
+
+using namespace XrdClTests;
+
+//------------------------------------------------------------------------------
+// Declaration
+//------------------------------------------------------------------------------
+class FileCopyTest : public ::testing::Test
+{
+  public:
+    void DownloadTestFunc();
+    void UploadTestFunc();
+    void CopyTestFunc( bool thirdParty = true );
+};
+
+//------------------------------------------------------------------------------
+// Download test
+//------------------------------------------------------------------------------
+void FileCopyTest::DownloadTestFunc()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string remoteFile;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "RemoteFile",    remoteFile ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string fileUrl = address + "/" + remoteFile;
+
+  const uint32_t  MB = 1024*1024;
+  char           *buffer = new char[4*MB];
+  StatInfo       *stat = nullptr;
+  File            f;
+
+  //----------------------------------------------------------------------------
+  // Open and stat the file
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.Open( fileUrl, OpenFlags::Read ) );
+
+  GTEST_ASSERT_XRDST( f.Stat( false, stat ) );
+  EXPECT_TRUE( stat );
+  EXPECT_TRUE( stat->TestFlags( StatInfo::IsReadable ) );
+
+  //----------------------------------------------------------------------------
+  // Fetch the data
+  //----------------------------------------------------------------------------
+  uint64_t    totalRead = 0;
+  uint32_t    bytesRead = 0;
+
+  CheckSumManager *man      = DefaultEnv::GetCheckSumManager();
+  EXPECT_TRUE( man );
+  XrdCksCalc      *crc32Sum = man->GetCalculator("zcrc32");
+  EXPECT_TRUE( crc32Sum );
+
+  while( 1 )
+  {
+    GTEST_ASSERT_XRDST( f.Read( totalRead, 4*MB, buffer, bytesRead ) );
+    if( bytesRead == 0 )
+      break;
+    totalRead += bytesRead;
+    crc32Sum->Update( buffer, bytesRead );
+  }
+
+  //----------------------------------------------------------------------------
+  // Compare the checksums
+  //----------------------------------------------------------------------------
+  char crcBuff[9];
+  XrdCksData crc; crc.Set( (const void *)crc32Sum->Final(), 4 ); crc.Get( crcBuff, 9 );
+  std::string transferSum = "zcrc32:"; transferSum += crcBuff;
+
+  std::string remoteSum;
+  std::string lastUrl;
+  EXPECT_TRUE( f.GetProperty( "LastURL", lastUrl ) );
+  GTEST_ASSERT_XRDST( Utils::GetRemoteCheckSum( remoteSum, "zcrc32",
+                                                  URL( lastUrl ) ) );
+  EXPECT_EQ( remoteSum, transferSum );
+
+  delete stat;
+  delete crc32Sum;
+  delete[] buffer;
+}
+
+//------------------------------------------------------------------------------
+// Upload test
+//------------------------------------------------------------------------------
+void FileCopyTest::UploadTestFunc()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+  std::string localFile;
+  std::string localDataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+  EXPECT_TRUE( testEnv->GetString( "LocalDataPath", localDataPath ) );
+  localFile = localDataPath + "/metaman/data/testFile.dat";
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string fileUrl = address + "/" + dataPath + "/testUpload.dat";
+  std::string remoteFile = dataPath + "/testUpload.dat";
+
+  const uint32_t  MB = 1024*1024;
+  char           *buffer = new char[4*MB];
+  File            f;
+
+  //----------------------------------------------------------------------------
+  // Open
+  //----------------------------------------------------------------------------
+  int fd = -1;
+
+  GTEST_ASSERT_ERRNO( (fd=open( localFile.c_str(), O_RDONLY )) > 0 )
+  GTEST_ASSERT_XRDST( f.Open( fileUrl,
+                                OpenFlags::Delete|OpenFlags::Update ) );
+
+  //----------------------------------------------------------------------------
+  // Read the data
+  //----------------------------------------------------------------------------
+  uint64_t offset        = 0;
+  ssize_t  bytesRead;
+
+  CheckSumManager *man      = DefaultEnv::GetCheckSumManager();
+  XrdCksCalc      *crc32Sum = man->GetCalculator("zcrc32");
+  EXPECT_TRUE( crc32Sum );
+
+  while( (bytesRead = read( fd, buffer, 4*MB )) > 0 )
+  {
+    crc32Sum->Update( buffer, bytesRead );
+    GTEST_ASSERT_XRDST( f.Write( offset, bytesRead, buffer ) );
+    offset += bytesRead;
+  }
+
+  EXPECT_GE( bytesRead, 0 );
+  close( fd );
+  GTEST_ASSERT_XRDST( f.Close() );
+  delete [] buffer;
+
+  //----------------------------------------------------------------------------
+  // Find out which server has the file
+  //----------------------------------------------------------------------------
+  FileSystem  fs( url );
+  LocationInfo *locations = nullptr;
+  GTEST_ASSERT_XRDST( fs.DeepLocate( remoteFile, OpenFlags::Refresh, locations ) );
+  EXPECT_TRUE( locations );
+  EXPECT_NE( locations->GetSize(), 0 );
+  FileSystem fs1( locations->Begin()->GetAddress() );
+  delete locations;
+
+  //----------------------------------------------------------------------------
+  // Verify the size
+  //----------------------------------------------------------------------------
+  StatInfo   *stat = nullptr;
+  GTEST_ASSERT_XRDST( fs1.Stat( remoteFile, stat ) );
+  EXPECT_TRUE( stat );
+  EXPECT_EQ( stat->GetSize(), offset );
+
+  //----------------------------------------------------------------------------
+  // Compare the checksums
+  //----------------------------------------------------------------------------
+  char crcBuff[9];
+  XrdCksData crc; crc.Set( (const void *)crc32Sum->Final(), 4 ); crc.Get( crcBuff, 9 );
+  std::string transferSum = "zcrc32:"; transferSum += crcBuff;
+
+  std::string remoteSum, lastUrl;
+  f.GetProperty( "LastURL", lastUrl );
+  GTEST_ASSERT_XRDST( Utils::GetRemoteCheckSum( remoteSum, "zcrc32",
+                                                  lastUrl ) );
+  EXPECT_EQ( remoteSum, transferSum );
+
+  //----------------------------------------------------------------------------
+  // Delete the file
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( fs.Rm( dataPath + "/testUpload.dat" ) );
+
+  delete stat;
+  delete crc32Sum;
+}
+
+//------------------------------------------------------------------------------
+// Upload test
+//------------------------------------------------------------------------------
+TEST_F(FileCopyTest, UploadTest)
+{
+  UploadTestFunc();
+}
+
+TEST_F(FileCopyTest, MultiStreamUploadTest)
+{
+  XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv();
+  env->PutInt( "SubStreamsPerChannel", 4 );
+  UploadTestFunc();
+}
+
+//------------------------------------------------------------------------------
+// Download test
+//------------------------------------------------------------------------------
+TEST_F(FileCopyTest, DownloadTest)
+{
+  DownloadTestFunc();
+}
+
+TEST_F(FileCopyTest, MultiStreamDownloadTest)
+{
+  XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv();
+  env->PutInt( "SubStreamsPerChannel", 4 );
+  DownloadTestFunc();
+}
+
+namespace
+{
+  //----------------------------------------------------------------------------
+  // Abort handler
+  //----------------------------------------------------------------------------
+  class CancelProgressHandler: public XrdCl::CopyProgressHandler
+  {
+    public:
+      //------------------------------------------------------------------------
+      // Constructor/destructor
+      //------------------------------------------------------------------------
+
+      // file size limit in MB
+      uint64_t sizeLimit;
+
+      CancelProgressHandler(): pCancel( false ) {
+        sizeLimit = 128*1024*1024;
+      }
+
+      CancelProgressHandler(uint64_t sl): pCancel( false ) {
+        sizeLimit = sl*1024*1024;
+      }
+
+      virtual ~CancelProgressHandler() {};
+
+      //------------------------------------------------------------------------
+      // Job progress
+      //------------------------------------------------------------------------
+      virtual void JobProgress( uint16_t jobNum,
+                                uint64_t bytesProcessed,
+                                uint64_t bytesTotal )
+      {
+        if( bytesProcessed > sizeLimit )
+          pCancel = true;
+      }
+
+      //------------------------------------------------------------------------
+      // Determine whether the job should be canceled
+      //------------------------------------------------------------------------
+      virtual bool ShouldCancel( uint16_t jobNum ) { return pCancel; }
+
+    private:
+      bool pCancel;
+  };
+}
+
+//------------------------------------------------------------------------------
+// Third party copy test
+//------------------------------------------------------------------------------
+void FileCopyTest::CopyTestFunc( bool thirdParty )
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string metamanager;
+  std::string manager1;
+  std::string manager2;
+  std::string sourceFile;
+  std::string dataPath;
+  std::string relativeDataPath;
+  std::string localDataPath;
+
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL",   metamanager ) );
+  EXPECT_TRUE( testEnv->GetString( "Manager1URL",        manager1 ) );
+  EXPECT_TRUE( testEnv->GetString( "Manager2URL",        manager2 ) );
+  EXPECT_TRUE( testEnv->GetString( "RemoteFile",       sourceFile ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath",           dataPath ) );
+  EXPECT_TRUE( testEnv->GetString( "LocalDataPath", relativeDataPath ) );
+
+  // getting the abs path to that it can work with the "file" protocol
+  localDataPath = realpath(relativeDataPath.c_str(), NULL);
+
+  std::string sourceURL    = manager1 + "/" + sourceFile;
+  std::string targetPath   = dataPath + "/tpcFile";
+  std::string targetURL    = manager2 + "/" + targetPath;
+  std::string metalinkURL  = metamanager + "/" + dataPath + "/metalink/mlTpcTest.meta4";
+  std::string metalinkURL2 = metamanager + "/" + dataPath + "/metalink/mlZipTest.meta4";
+  std::string zipURL       = metamanager + "/" + dataPath + "/data.zip";
+  std::string zipURL2      = metamanager + "/" + dataPath + "/large.zip";
+  std::string fileInZip    = "paper.txt";
+  std::string fileInZip2   = "bible.txt";
+  std::string xcpSourceURL = metamanager + "/" + dataPath + "/1db882c8-8cd6-4df1-941f-ce669bad3458.dat";
+  std::string localFile    = localDataPath + "/metaman/localfile.dat";
+
+  CopyProcess  process1, process2, process3, process4, process5, process6, process7, process8, process9,
+               process10, process11, process12, process13, process14, process15, process16, process17;
+  PropertyList properties, results;
+  FileSystem fs( manager2 );
+
+  //----------------------------------------------------------------------------
+  // Copy from a ZIP archive
+  //----------------------------------------------------------------------------
+  if( !thirdParty )
+  {
+    results.Clear();
+    properties.Set( "source",       zipURL    );
+    properties.Set( "target",       targetURL );
+    properties.Set( "zipArchive",   true      );
+    properties.Set( "zipSource",    fileInZip );
+    GTEST_ASSERT_XRDST( process6.AddJob( properties, &results ) );
+    GTEST_ASSERT_XRDST( process6.Prepare() );
+    GTEST_ASSERT_XRDST( process6.Run(0) );
+    GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+    properties.Clear();
+
+    //--------------------------------------------------------------------------
+    // Copy from a ZIP archive (compressed) and validate the zcrc32 checksum
+    //--------------------------------------------------------------------------
+    results.Clear();
+    properties.Set( "source",       zipURL2 );
+    properties.Set( "target",       targetURL );
+    properties.Set( "checkSumMode",   "end2end"     );
+    properties.Set( "checkSumType",   "zcrc32"      );
+    properties.Set( "zipArchive",   true      );
+    properties.Set( "zipSource",    fileInZip2 );
+    GTEST_ASSERT_XRDST( process10.AddJob( properties, &results ) );
+    GTEST_ASSERT_XRDST( process10.Prepare() );
+    GTEST_ASSERT_XRDST( process10.Run(0) );
+    GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+    properties.Clear();
+
+    //--------------------------------------------------------------------------
+    // Copy with `--rm-bad-cksum`
+    //--------------------------------------------------------------------------
+    results.Clear();
+    properties.Set( "source",         sourceURL );
+    properties.Set( "target",         targetURL );
+    properties.Set( "checkSumMode",   "end2end" );
+    properties.Set( "checkSumType",   "auto"    );
+    properties.Set( "checkSumPreset", "bad-value" ); //< provide wrong checksum value, so the check fails and the file gets removed
+    properties.Set( "rmOnBadCksum",   true        );
+    GTEST_ASSERT_XRDST( process12.AddJob( properties, &results ) );
+    GTEST_ASSERT_XRDST( process12.Prepare() );
+    GTEST_ASSERT_XRDST_NOTOK( process12.Run(0), XrdCl::errCheckSumError );
+    XrdCl::StatInfo *info = nullptr;
+    GTEST_ASSERT_XRDST_NOTOK( fs.Stat( targetPath, info ), XrdCl::errErrorResponse );
+    properties.Clear();
+
+    //--------------------------------------------------------------------------
+    // Copy with `--zip-mtln-cksum`
+    //--------------------------------------------------------------------------
+    results.Clear();
+    properties.Set( "source",         metalinkURL2 );
+    properties.Set( "target",         targetURL    );
+    properties.Set( "checkSumMode",   "end2end"    );
+    properties.Set( "checkSumType",   "zcrc32"     );
+    XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv();
+    env->PutInt( "ZipMtlnCksum", 1 );
+    GTEST_ASSERT_XRDST( process13.AddJob( properties, &results ) );
+    GTEST_ASSERT_XRDST( process13.Prepare() );
+    GTEST_ASSERT_XRDST_NOTOK( process13.Run(0), XrdCl::errCheckSumError );
+    env->PutInt( "ZipMtlnCksum", 0 );
+    GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+
+    //--------------------------------------------------------------------------
+    // Copy with
+    //   `--xrate`
+    //   `--xrate-threshold`
+    //--------------------------------------------------------------------------
+    results.Clear();
+    properties.Clear();
+    properties.Set( "source",          sourceURL        );
+    properties.Set( "target",          targetURL        );
+    properties.Set( "xrate",           1024 * 1024 * 32 ); //< limit the transfer rate to 32MB/s
+    properties.Set( "xrateThreshold", 1024 * 1024 * 30 ); //< fail the job if the transfer rate drops under 30MB/s
+    GTEST_ASSERT_XRDST( process14.AddJob( properties, &results ) );
+    GTEST_ASSERT_XRDST( process14.Prepare() );
+    GTEST_ASSERT_XRDST( process14.Run(0) );
+    GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+
+    //--------------------------------------------------------------------------
+    // Now test the cp-timeout
+    //--------------------------------------------------------------------------
+    results.Clear();
+    properties.Clear();
+    properties.Set( "source",    sourceURL   );
+    properties.Set( "target",    targetURL   );
+    properties.Set( "xrate",     1024 * 1024 ); //< limit the transfer rate to 1MB/s (the file is 1GB big so the transfer will take 1024 seconds)
+    properties.Set( "cpTimeout", 5          ); //< timeout the job after 10 seconds (now the file are smaller so we have to decrease it to 5 sec)
+    GTEST_ASSERT_XRDST( process15.AddJob( properties, &results ) );
+    GTEST_ASSERT_XRDST( process15.Prepare() );
+    GTEST_ASSERT_XRDST_NOTOK( process15.Run(0), XrdCl::errOperationExpired );
+    GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+
+    //--------------------------------------------------------------------------
+    // Test posc for local files
+    //--------------------------------------------------------------------------
+    results.Clear();
+    properties.Clear();
+    std::string localtrg = "file://localhost" + localDataPath + "/metaman/tpcFile.dat";
+    properties.Set( "source",    sourceURL );
+    properties.Set( "target",    localtrg  );
+    properties.Set( "posc",      true      );
+    CancelProgressHandler progress16(5); //> abort the copy after 5MB
+    GTEST_ASSERT_XRDST( process16.AddJob( properties, &results ) );
+    GTEST_ASSERT_XRDST( process16.Prepare() );
+    GTEST_ASSERT_XRDST_NOTOK( process16.Run( &progress16 ), errOperationInterrupted );
+    XrdCl::FileSystem localfs( "file://localhost" );
+    XrdCl::StatInfo *ptr = nullptr;
+    GTEST_ASSERT_XRDST_NOTOK( localfs.Stat( dataPath + "/tpcFile.dat", ptr ), XrdCl::errLocalError );
+
+    //--------------------------------------------------------------------------
+    // Test --retry and --retry-policy
+    //--------------------------------------------------------------------------
+    results.Clear();
+    properties.Clear();
+    properties.Set( "xrate",     1024 * 1024 * 32 ); //< limit the transfer rate to 32MB/s
+    properties.Set( "cpTimeout", 20               ); //< timeout the job after 20 seconds
+    properties.Set( "source",    sourceURL        );
+    properties.Set( "target",    targetURL        );
+    env->PutInt( "CpRetry", 1 );
+    env->PutString( "CpRetryPolicy", "continue" );
+    GTEST_ASSERT_XRDST( process17.AddJob( properties, &results ) );
+    GTEST_ASSERT_XRDST( process17.Prepare() );
+    GTEST_ASSERT_XRDST( process17.Run(0) );
+    GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+    env->PutInt( "CpRetry", XrdCl::DefaultCpRetry );
+    env->PutString( "CpRetryPolicy", XrdCl::DefaultCpRetryPolicy );
+  }
+
+  //----------------------------------------------------------------------------
+  // Copy from a Metalink
+  //----------------------------------------------------------------------------
+  results.Clear();
+  properties.Clear();
+  properties.Set( "source",       metalinkURL );
+  properties.Set( "target",       targetURL   );
+  properties.Set( "checkSumMode", "end2end"   );
+  properties.Set( "checkSumType", "crc32c"    );
+  GTEST_ASSERT_XRDST( process5.AddJob( properties, &results ) );
+  GTEST_ASSERT_XRDST( process5.Prepare() );
+  GTEST_ASSERT_XRDST( process5.Run(0) );
+  GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+  properties.Clear();
+
+  // XCp test
+  results.Clear();
+  properties.Set( "source",         xcpSourceURL  );
+  properties.Set( "target",         targetURL     );
+  properties.Set( "checkSumMode",   "end2end"     );
+  properties.Set( "checkSumType",   "crc32c"      );
+  properties.Set( "xcp",            true          );
+  properties.Set( "nbXcpSources",   3             );
+  GTEST_ASSERT_XRDST( process7.AddJob( properties, &results ) );
+  GTEST_ASSERT_XRDST( process7.Prepare() );
+  GTEST_ASSERT_XRDST( process7.Run(0) );
+  GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+  properties.Clear();
+
+  //----------------------------------------------------------------------------
+  // Copy to local fs
+  //----------------------------------------------------------------------------
+  results.Clear();
+  properties.Set( "source", sourceURL );
+  properties.Set( "target", "file://localhost" + localFile );
+  properties.Set( "checkSumMode", "end2end" );
+  properties.Set( "checkSumType", "crc32c"  );
+  GTEST_ASSERT_XRDST( process8.AddJob( properties, &results ) );
+  GTEST_ASSERT_XRDST( process8.Prepare() );
+  GTEST_ASSERT_XRDST( process8.Run(0) );
+  properties.Clear();
+
+  //----------------------------------------------------------------------------
+  // Copy from local fs with extended attributes
+  //----------------------------------------------------------------------------
+
+  // set extended attributes in the local source file
+  File lf;
+  GTEST_ASSERT_XRDST( lf.Open( "file://localhost" + localFile, OpenFlags::Write ) );
+  std::vector attrs; attrs.push_back( xattr_t( "foo", "bar" ) );
+  std::vector result;
+  GTEST_ASSERT_XRDST( lf.SetXAttr( attrs, result ) );
+  EXPECT_EQ( result.size(), 1 );
+  GTEST_ASSERT_XRDST( result.front().status );
+  GTEST_ASSERT_XRDST( lf.Close() );
+
+  results.Clear();
+  properties.Set( "source", "file://localhost" + localFile );
+  properties.Set( "target", targetURL );
+  properties.Set( "checkSumMode", "end2end" );
+  properties.Set( "checkSumType", "crc32c"  );
+  properties.Set( "preserveXAttr", true );
+  GTEST_ASSERT_XRDST( process9.AddJob( properties, &results ) );
+  GTEST_ASSERT_XRDST( process9.Prepare() );
+  GTEST_ASSERT_XRDST( process9.Run(0) );
+  properties.Clear();
+
+  // now test if the xattrs were preserved
+  std::vector xattrs;
+  GTEST_ASSERT_XRDST( fs.ListXAttr( targetPath, xattrs ) );
+  EXPECT_EQ( xattrs.size(), 1 );
+  XAttr &xattr = xattrs.front();
+  GTEST_ASSERT_XRDST( xattr.status );
+  EXPECT_EQ( xattr.name, "foo" );
+  EXPECT_EQ( xattr.value, "bar" );
+
+  //----------------------------------------------------------------------------
+  // Cleanup
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+  EXPECT_EQ( remove( localFile.c_str() ), 0 );
+
+  //----------------------------------------------------------------------------
+  // Initialize and run the copy
+  //----------------------------------------------------------------------------
+  properties.Set( "source",       sourceURL );
+  properties.Set( "target",       targetURL );
+  properties.Set( "checkSumMode", "end2end" );
+  properties.Set( "checkSumType", "crc32c"  );
+  if( thirdParty )
+    properties.Set( "thirdParty",   "only"    );
+  GTEST_ASSERT_XRDST( process1.AddJob( properties, &results ) );
+  GTEST_ASSERT_XRDST( process1.Prepare() );
+  GTEST_ASSERT_XRDST( process1.Run(0) );
+  GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+  properties.Clear();
+
+  //----------------------------------------------------------------------------
+  // Copy with `auto` checksum
+  //----------------------------------------------------------------------------
+  results.Clear();
+  properties.Set( "source",       sourceURL );
+  properties.Set( "target",       targetURL );
+  properties.Set( "checkSumMode", "end2end" );
+  properties.Set( "checkSumType", "auto"    );
+  if( thirdParty )
+    properties.Set( "thirdParty",   "only"    );
+  GTEST_ASSERT_XRDST( process11.AddJob( properties, &results ) );
+  GTEST_ASSERT_XRDST( process11.Prepare() );
+  GTEST_ASSERT_XRDST( process11.Run(0) );
+  GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+  properties.Clear();
+
+  // the further tests are only valid for third party copy for now
+  if( !thirdParty )
+    return;
+
+  //----------------------------------------------------------------------------
+  // Abort the copy after 100MB
+  //----------------------------------------------------------------------------
+//  CancelProgressHandler progress;
+//  GTEST_ASSERT_XRDST( process2.AddJob( properties, &results ) );
+//  GTEST_ASSERT_XRDST( process2.Prepare() );
+//  GTEST_ASSERT_XRDST_NOTOK( process2.Run(&progress), errErrorResponse );
+//  GTEST_ASSERT_XRDST( fs.Rm( targetPath ) );
+
+  //----------------------------------------------------------------------------
+  // Copy from a non-existent source
+  //----------------------------------------------------------------------------
+  results.Clear();
+  properties.Set( "source",      "root://localhost:9999//test" );
+  properties.Set( "target",      targetURL );
+  properties.Set( "initTimeout", 10 );
+  properties.Set( "thirdParty",  "only"    );
+  GTEST_ASSERT_XRDST( process3.AddJob( properties, &results ) );
+  GTEST_ASSERT_XRDST( process3.Prepare() );
+  XrdCl::XRootDStatus status = process3.Run(0);
+  EXPECT_TRUE( !status.IsOK() && ( status.code == errOperationExpired || status.code == errConnectionError ) );
+
+  //----------------------------------------------------------------------------
+  // Copy to a non-existent target
+  //----------------------------------------------------------------------------
+  results.Clear();
+  properties.Set( "source",      sourceURL );
+  properties.Set( "target",      "root://localhost:9997//test" ); // was 9999, this change allows for
+  properties.Set( "initTimeout", 10 );                            // parallel testing
+  properties.Set( "thirdParty",  "only"    );
+  GTEST_ASSERT_XRDST( process4.AddJob( properties, &results ) );
+  GTEST_ASSERT_XRDST( process4.Prepare() );
+  status = process4.Run(0);
+  EXPECT_TRUE( !status.IsOK() && ( status.code == errOperationExpired || status.code == errConnectionError ) );
+}
+
+//------------------------------------------------------------------------------
+// Third party copy test
+//------------------------------------------------------------------------------
+TEST_F(FileCopyTest, ThirdPartyCopyTest)
+{
+  CopyTestFunc( true );
+}
+
+//------------------------------------------------------------------------------
+// Normal copy test
+//------------------------------------------------------------------------------
+TEST_F(FileCopyTest, NormalCopyTest)
+{
+  CopyTestFunc( false );
+}
diff --git a/tests/XrdCl/XrdClFileSystemTest.cc b/tests/XrdCl/XrdClFileSystemTest.cc
new file mode 100644
index 00000000000..51a3082d4f7
--- /dev/null
+++ b/tests/XrdCl/XrdClFileSystemTest.cc
@@ -0,0 +1,758 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+#include 
+#include 
+#include "XrdCl/XrdClDefaultEnv.hh"
+#include "XrdCl/XrdClPlugInManager.hh"
+#include "GTestXrdHelpers.hh"
+
+#include 
+#include 
+#include "TestEnv.hh"
+#include "IdentityPlugIn.hh"
+
+using namespace XrdClTests;
+
+//------------------------------------------------------------------------------
+// Class declaration
+//------------------------------------------------------------------------------
+class FileSystemTest: public ::testing::Test
+{
+  public:
+    void LocateTest();
+    void MvTest();
+    void ServerQueryTest();
+    void TruncateRmTest();
+    void MkdirRmdirTest();
+    void ChmodTest();
+    void PingTest();
+    void StatTest();
+    void StatVFSTest();
+    void ProtocolTest();
+    void DeepLocateTest();
+    void DirListTest();
+    void SendInfoTest();
+    void PrepareTest();
+    void XAttrTest();
+};
+
+//------------------------------------------------------------------------------
+// Tests declaration
+//------------------------------------------------------------------------------
+TEST_F(FileSystemTest, LocateTest)
+{
+  LocateTest();
+}
+
+TEST_F(FileSystemTest, MvTest)
+{
+  MvTest();
+}
+
+TEST_F(FileSystemTest, ServerQueryTest)
+{
+  ServerQueryTest();
+}
+
+TEST_F(FileSystemTest, TruncateRmTest)
+{
+  TruncateRmTest();
+}
+
+TEST_F(FileSystemTest, MkdirRmdirTest)
+{
+  MkdirRmdirTest();
+}
+
+TEST_F(FileSystemTest, ChmodTest)
+{
+  ChmodTest();
+}
+
+TEST_F(FileSystemTest, PingTest)
+{
+  PingTest();
+}
+
+TEST_F(FileSystemTest, StatTest)
+{
+  StatTest();
+}
+
+TEST_F(FileSystemTest, StatVFSTest)
+{
+  StatVFSTest();
+}
+
+TEST_F(FileSystemTest, ProtocolTest)
+{
+  ProtocolTest();
+}
+
+TEST_F(FileSystemTest, DeepLocateTest)
+{
+  DeepLocateTest();
+}
+
+TEST_F(FileSystemTest, DirListTest)
+{
+  DirListTest();
+}
+
+TEST_F(FileSystemTest, SendInfoTest)
+{
+  SendInfoTest();
+}
+
+TEST_F(FileSystemTest, PrepareTest)
+{
+  PrepareTest();
+}
+
+TEST_F(FileSystemTest, XAttrTest)
+{
+  XAttrTest();
+}
+
+TEST_F(FileSystemTest, PlugInTest)
+{
+  XrdCl::PlugInFactory *f = new IdentityFactory;
+  XrdCl::DefaultEnv::GetPlugInManager()->RegisterDefaultFactory(f);
+  LocateTest();
+  MvTest();
+  ServerQueryTest();
+  TruncateRmTest();
+  MkdirRmdirTest();
+  ChmodTest();
+  PingTest();
+  StatTest();
+  StatVFSTest();
+  ProtocolTest();
+  DeepLocateTest();
+  DirListTest();
+  SendInfoTest();
+  PrepareTest();
+  XrdCl::DefaultEnv::GetPlugInManager()->RegisterDefaultFactory(0);
+}
+
+
+
+//------------------------------------------------------------------------------
+// Locate test
+//------------------------------------------------------------------------------
+void FileSystemTest::LocateTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string remoteFile;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "RemoteFile", remoteFile ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  //----------------------------------------------------------------------------
+  // Query the server for all of the file locations
+  //----------------------------------------------------------------------------
+  FileSystem fs( url );
+
+  LocationInfo *locations = 0;
+  GTEST_ASSERT_XRDST( fs.Locate( remoteFile, OpenFlags::Refresh, locations ) );
+  EXPECT_TRUE( locations );
+  EXPECT_NE( locations->GetSize(), 0 );
+  delete locations;
+}
+
+//------------------------------------------------------------------------------
+// Mv test
+//------------------------------------------------------------------------------
+void FileSystemTest::MvTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+  std::string remoteFile;
+
+  EXPECT_TRUE( testEnv->GetString( "DiskServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "RemoteFile",    remoteFile ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string filePath1 = remoteFile;
+  std::string filePath2 = remoteFile + "2";
+
+
+  LocationInfo *info = 0;
+  FileSystem fs( url );
+
+  // move the file
+  GTEST_ASSERT_XRDST( fs.Mv( filePath1, filePath2 ) );
+  // make sure it's not there
+  GTEST_ASSERT_XRDST_NOTOK( fs.Locate( filePath1, OpenFlags::Refresh, info ),
+                              errErrorResponse );
+  // make sure the destination is there
+  GTEST_ASSERT_XRDST( fs.Locate( filePath2, OpenFlags::Refresh, info ) );
+  delete info;
+  // move it back
+  GTEST_ASSERT_XRDST( fs.Mv( filePath2, filePath1 ) );
+  // make sure it's there
+  GTEST_ASSERT_XRDST( fs.Locate( filePath1, OpenFlags::Refresh, info ) );
+  delete info;
+  // make sure the other one is gone
+  GTEST_ASSERT_XRDST_NOTOK( fs.Locate( filePath2, OpenFlags::Refresh, info ),
+                              errErrorResponse );
+}
+
+//------------------------------------------------------------------------------
+// Query test
+//------------------------------------------------------------------------------
+void FileSystemTest::ServerQueryTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string remoteFile;
+
+  EXPECT_TRUE( testEnv->GetString( "DiskServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "RemoteFile", remoteFile ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  FileSystem fs( url );
+  Buffer *response = 0;
+  Buffer  arg;
+  arg.FromString( remoteFile );
+  GTEST_ASSERT_XRDST( fs.Query( QueryCode::Checksum, arg, response ) );
+  EXPECT_TRUE( response );
+  EXPECT_NE( response->GetSize(), 0 );
+  delete response;
+}
+
+//------------------------------------------------------------------------------
+// Truncate/Rm test
+//------------------------------------------------------------------------------
+void FileSystemTest::TruncateRmTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string filePath = dataPath + "/testfile";
+  std::string fileUrl  = address + "/";
+  fileUrl += filePath;
+
+  FileSystem fs( url );
+  File       f;
+  GTEST_ASSERT_XRDST( f.Open( fileUrl, OpenFlags::Update | OpenFlags::Delete,
+                                Access::UR | Access::UW ) );
+  GTEST_ASSERT_XRDST( fs.Truncate( filePath, 10000000 ) );
+  GTEST_ASSERT_XRDST( fs.Rm( filePath ) );
+}
+
+//------------------------------------------------------------------------------
+// Mkdir/Rmdir test
+//------------------------------------------------------------------------------
+void FileSystemTest::MkdirRmdirTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "DiskServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string dirPath1 = dataPath + "/testdir";
+  std::string dirPath2 = dataPath + "/testdir/asdads";
+
+  FileSystem fs( url );
+
+  GTEST_ASSERT_XRDST( fs.MkDir( dirPath2, MkDirFlags::MakePath,
+                              Access::UR | Access::UW | Access::UX ) );
+  GTEST_ASSERT_XRDST( fs.RmDir( dirPath2 ) );
+  GTEST_ASSERT_XRDST( fs.RmDir( dirPath1 ) );
+}
+
+//------------------------------------------------------------------------------
+// Chmod test
+//------------------------------------------------------------------------------
+void FileSystemTest::ChmodTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "DiskServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string dirPath = dataPath + "/testdir";
+
+  FileSystem fs( url );
+
+  GTEST_ASSERT_XRDST( fs.MkDir( dirPath, MkDirFlags::MakePath,
+                                  Access::UR | Access::UW | Access::UX ) );
+  GTEST_ASSERT_XRDST( fs.ChMod( dirPath,
+                                  Access::UR | Access::UW | Access::UX |
+                                  Access::GR | Access::GX ) );
+  GTEST_ASSERT_XRDST( fs.RmDir( dirPath ) );
+}
+
+//------------------------------------------------------------------------------
+// Locate test
+//------------------------------------------------------------------------------
+void FileSystemTest::PingTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  FileSystem fs( url );
+  GTEST_ASSERT_XRDST( fs.Ping() );
+}
+
+//------------------------------------------------------------------------------
+// Stat test
+//------------------------------------------------------------------------------
+void FileSystemTest::StatTest()
+{
+  using namespace XrdCl;
+
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string remoteFile;
+  std::string localDataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "RemoteFile",    remoteFile ) );
+  EXPECT_TRUE( testEnv->GetString( "LocalDataPath", localDataPath ) );
+
+  std::string localFilePath = localDataPath + "/srv1" + remoteFile;
+
+  struct stat localStatBuf;
+  int rc = stat(localFilePath.c_str(), &localStatBuf);
+  EXPECT_EQ( rc, 0 );
+  uint64_t fileSize = localStatBuf.st_size;
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  FileSystem fs( url );
+  StatInfo *response = 0;
+  GTEST_ASSERT_XRDST( fs.Stat( remoteFile, response ) );
+  EXPECT_TRUE( response );
+  EXPECT_EQ( response->GetSize(), fileSize );
+  EXPECT_TRUE( response->TestFlags( StatInfo::IsReadable ) );
+  EXPECT_TRUE( response->TestFlags( StatInfo::IsWritable ) );
+  EXPECT_TRUE( !response->TestFlags( StatInfo::IsDir ) );
+  delete response;
+}
+
+//------------------------------------------------------------------------------
+// Stat VFS test
+//------------------------------------------------------------------------------
+void FileSystemTest::StatVFSTest()
+{
+  using namespace XrdCl;
+
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  FileSystem fs( url );
+  StatInfoVFS *response = 0;
+  GTEST_ASSERT_XRDST( fs.StatVFS( dataPath, response ) );
+  EXPECT_TRUE( response );
+  delete response;
+}
+
+//------------------------------------------------------------------------------
+// Protocol test
+//------------------------------------------------------------------------------
+void FileSystemTest::ProtocolTest()
+{
+  using namespace XrdCl;
+
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  FileSystem fs( url );
+  ProtocolInfo *response = 0;
+  GTEST_ASSERT_XRDST( fs.Protocol( response ) );
+  EXPECT_TRUE( response );
+  delete response;
+}
+
+//------------------------------------------------------------------------------
+// Deep locate test
+//------------------------------------------------------------------------------
+void FileSystemTest::DeepLocateTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string remoteFile;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "RemoteFile",    remoteFile ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  //----------------------------------------------------------------------------
+  // Query the server for all of the file locations
+  //----------------------------------------------------------------------------
+  FileSystem fs( url );
+
+  LocationInfo *locations = 0;
+  GTEST_ASSERT_XRDST( fs.DeepLocate( remoteFile, OpenFlags::Refresh, locations ) );
+  EXPECT_TRUE( locations );
+  EXPECT_NE( locations->GetSize(), 0 );
+  LocationInfo::Iterator it = locations->Begin();
+  for( ; it != locations->End(); ++it )
+    EXPECT_TRUE( it->IsServer() );
+  delete locations;
+}
+
+//------------------------------------------------------------------------------
+// Dir list
+//------------------------------------------------------------------------------
+void FileSystemTest::DirListTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string lsPath = dataPath + "/bigdir";
+
+  //----------------------------------------------------------------------------
+  // Query the server for all of the file locations
+  //----------------------------------------------------------------------------
+  FileSystem fs( url );
+
+  DirectoryList *list = 0;
+  GTEST_ASSERT_XRDST( fs.DirList( lsPath, DirListFlags::Stat | DirListFlags::Locate, list ) );
+  EXPECT_TRUE( list );
+  EXPECT_EQ( list->GetSize(), 4000 );
+
+  std::set dirls1;
+  for( auto itr = list->Begin(); itr != list->End(); ++itr )
+  {
+    DirectoryList::ListEntry *entry = *itr;
+    dirls1.insert( entry->GetName() );
+  }
+
+  delete list;
+  list = 0;
+
+  //----------------------------------------------------------------------------
+  // Now do a chunked query
+  //----------------------------------------------------------------------------
+  std::set dirls2;
+
+  LocationInfo *info = 0;
+  GTEST_ASSERT_XRDST( fs.DeepLocate( lsPath, OpenFlags::PrefName, info ) );
+  EXPECT_TRUE( info );
+
+  for( auto itr = info->Begin(); itr != info->End(); ++itr )
+  {
+    XrdSysSemaphore sem( 0 );
+    auto handler = XrdCl::ResponseHandler::Wrap( [&]( auto &s, auto &r )
+      {
+        GTEST_ASSERT_XRDST( s );
+        auto &list = To( r );
+        for( auto itr = list.Begin(); itr != list.End(); ++itr )
+          dirls2.insert( ( *itr )->GetName() );
+        if( s.code == XrdCl::suDone )
+          sem.Post();
+      } );
+
+    FileSystem fs1( std::string( itr->GetAddress() ) );
+    GTEST_ASSERT_XRDST( fs1.DirList( lsPath, DirListFlags::Stat | DirListFlags::Chunked, handler ) );
+    sem.Wait();
+  }
+  delete info;
+  info = 0;
+
+  EXPECT_EQ( dirls1, dirls2 );
+
+  //----------------------------------------------------------------------------
+  // Now list an empty directory
+  //----------------------------------------------------------------------------
+  std::string emptyDirPath = dataPath + "/empty" ;
+  GTEST_ASSERT_XRDST( fs.MkDir( emptyDirPath, MkDirFlags::None, Access::None ) );
+  GTEST_ASSERT_XRDST( fs.DeepLocate( emptyDirPath, OpenFlags::PrefName, info ) );
+  EXPECT_TRUE( info->GetSize() );
+  FileSystem fs3( info->Begin()->GetAddress() );
+  GTEST_ASSERT_XRDST( fs3.DirList( emptyDirPath, DirListFlags::Stat, list ) );
+  EXPECT_TRUE( list );
+  EXPECT_EQ( list->GetSize(), 0 );
+  GTEST_ASSERT_XRDST( fs.RmDir( emptyDirPath ) );
+
+  delete list;
+  list = 0;
+  delete info;
+  info = 0;
+}
+
+
+//------------------------------------------------------------------------------
+// Set
+//------------------------------------------------------------------------------
+void FileSystemTest::SendInfoTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  FileSystem fs( url );
+
+  Buffer *id = 0;
+  GTEST_ASSERT_XRDST( fs.SendInfo( "test stuff", id ) );
+  EXPECT_TRUE( id );
+  EXPECT_EQ( id->GetSize(), 4 );
+  delete id;
+}
+
+
+//------------------------------------------------------------------------------
+// Set
+//------------------------------------------------------------------------------
+void FileSystemTest::PrepareTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  FileSystem fs( url );
+
+  Buffer *id = 0;
+  std::vector list;
+
+  std::string fileLocation = dataPath + "/1db882c8-8cd6-4df1-941f-ce669bad3458.dat";
+  list.push_back( fileLocation );
+  list.push_back( fileLocation );
+
+  GTEST_ASSERT_XRDST( fs.Prepare( list, PrepareFlags::Stage, 1, id ) );
+  EXPECT_TRUE( id );
+  EXPECT_TRUE( id->GetSize() );
+  delete id;
+}
+
+//------------------------------------------------------------------------------
+// Extended attributes test
+//------------------------------------------------------------------------------
+void FileSystemTest::XAttrTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string remoteFile;
+
+  EXPECT_TRUE( testEnv->GetString( "DiskServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "RemoteFile",    remoteFile ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  FileSystem fs( url );
+
+  std::map attributes
+  {
+      std::make_pair( "version",  "v1.2.3-45" ),
+      std::make_pair( "checksum", "2ccc0e85556a6cd193dd8d2b40aab50c" ),
+      std::make_pair( "index",    "4" )
+  };
+
+  //----------------------------------------------------------------------------
+  // Test SetXAttr
+  //----------------------------------------------------------------------------
+  std::vector attrs;
+  auto itr1 = attributes.begin();
+  for( ; itr1 != attributes.end() ; ++itr1 )
+    attrs.push_back( std::make_tuple( itr1->first, itr1->second ) );
+
+  std::vector result1;
+  GTEST_ASSERT_XRDST( fs.SetXAttr( remoteFile, attrs, result1 ) );
+
+  auto itr2 = result1.begin();
+  for( ; itr2 != result1.end() ; ++itr2 )
+    GTEST_ASSERT_XRDST( itr2->status );
+  result1.clear();
+
+  //----------------------------------------------------------------------------
+  // Test GetXAttr
+  //----------------------------------------------------------------------------
+  std::vector names;
+  itr1 = attributes.begin();
+  for( ; itr1 != attributes.end() ; ++itr1 )
+    names.push_back( itr1->first );
+
+  std::vector result2;
+  GTEST_ASSERT_XRDST( fs.GetXAttr( remoteFile, names, result2 ) );
+
+  auto itr3 = result2.begin();
+  for( ; itr3 != result2.end() ; ++itr3 )
+  {
+    GTEST_ASSERT_XRDST( itr3->status );
+    auto match = attributes.find( itr3->name );
+    EXPECT_NE( match, attributes.end() );
+    EXPECT_EQ( match->second, itr3->value );
+  }
+  result2.clear();
+
+  //----------------------------------------------------------------------------
+  // Test ListXAttr
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( fs.ListXAttr( remoteFile, result2 ) );
+
+  itr3 = result2.begin();
+  for( ; itr3 != result2.end() ; ++itr3 )
+  {
+    GTEST_ASSERT_XRDST( itr3->status );
+    auto match = attributes.find( itr3->name );
+    EXPECT_NE( match, attributes.end() );
+    EXPECT_EQ( match->second, itr3->value );
+  }
+
+  result2.clear();
+
+  //----------------------------------------------------------------------------
+  // Test DelXAttr
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( fs.DelXAttr( remoteFile, names, result1 ) );
+
+  itr2 = result1.begin();
+  for( ; itr2 != result1.end() ; ++itr2 )
+    GTEST_ASSERT_XRDST( itr2->status );
+
+  result1.clear();
+}
+
diff --git a/tests/XrdCl/XrdClFileTest.cc b/tests/XrdCl/XrdClFileTest.cc
new file mode 100644
index 00000000000..8c7f9bcbfed
--- /dev/null
+++ b/tests/XrdCl/XrdClFileTest.cc
@@ -0,0 +1,917 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+#include "TestEnv.hh"
+#include "Utils.hh"
+#include "IdentityPlugIn.hh"
+
+#include "GTestXrdHelpers.hh"
+#include "XrdCl/XrdClFile.hh"
+#include "XrdCl/XrdClDefaultEnv.hh"
+#include "XrdCl/XrdClPlugInManager.hh"
+#include "XrdCl/XrdClMessage.hh"
+#include "XrdCl/XrdClSIDManager.hh"
+#include "XrdCl/XrdClPostMaster.hh"
+#include "XrdCl/XrdClXRootDTransport.hh"
+#include "XrdCl/XrdClMessageUtils.hh"
+#include "XrdCl/XrdClXRootDMsgHandler.hh"
+#include "XrdCl/XrdClCopyProcess.hh"
+#include "XrdCl/XrdClZipArchive.hh"
+#include "XrdCl/XrdClConstants.hh"
+#include "XrdCl/XrdClZipOperations.hh"
+#include 
+#include 
+#include 
+
+using namespace XrdClTests;
+
+//------------------------------------------------------------------------------
+// Class declaration
+//------------------------------------------------------------------------------
+class FileTest: public ::testing::Test
+{
+  public:
+    void RedirectReturnTest();
+    void ReadTest();
+    void WriteTest();
+    void WriteVTest();
+    void VectorReadTest();
+    void VectorWriteTest();
+    void VirtualRedirectorTest();
+    void XAttrTest();
+};
+
+//------------------------------------------------------------------------------
+// Tests declaration
+//------------------------------------------------------------------------------
+TEST_F(FileTest, RedirectReturnTest)
+{
+  RedirectReturnTest();
+}
+
+TEST_F(FileTest, ReadTest)
+{
+  ReadTest();
+}
+
+TEST_F(FileTest, WriteTest)
+{
+  WriteTest();
+}
+
+TEST_F(FileTest, WriteVTest)
+{
+  WriteVTest();
+}
+
+TEST_F(FileTest, VectorReadTest)
+{
+  VectorReadTest();
+}
+
+TEST_F(FileTest, VectorWriteTest)
+{
+  VectorWriteTest();
+}
+
+// This test is commented out as of now since we think that there's a bug in the code
+
+// TEST_F(FileTest, VirtualRedirectorTest)
+// {
+//   VirtualRedirectorTest();
+// }
+
+TEST_F(FileTest, XAttrTest)
+{
+  XAttrTest();
+}
+
+TEST_F(FileTest, PlugInTest)
+{
+  XrdCl::PlugInFactory *f = new IdentityFactory;
+  XrdCl::DefaultEnv::GetPlugInManager()->RegisterDefaultFactory(f);
+  RedirectReturnTest();
+  ReadTest();
+  WriteTest();
+  VectorReadTest();
+  XrdCl::DefaultEnv::GetPlugInManager()->RegisterDefaultFactory(0);
+}
+
+//------------------------------------------------------------------------------
+// Redirect return test
+//------------------------------------------------------------------------------
+void FileTest::RedirectReturnTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string path = dataPath + "/cb4aacf1-6f28-42f2-b68a-90a73460f424.dat";
+  std::string fileUrl = address + "/" + path;
+
+  //----------------------------------------------------------------------------
+  // Build the open request
+  //----------------------------------------------------------------------------
+  Message           *msg;
+  ClientOpenRequest *req;
+  MessageUtils::CreateRequest( msg, req, path.length() );
+  req->requestid = kXR_open;
+  req->options   = kXR_open_read | kXR_retstat;
+  req->dlen      = path.length();
+  msg->Append( path.c_str(), path.length(), 24 );
+  XRootDTransport::SetDescription( msg );
+
+  SyncResponseHandler *handler = new SyncResponseHandler();
+  MessageSendParams params; params.followRedirects = false;
+  MessageUtils::ProcessSendParams( params );
+  OpenInfo *response = 0;
+  GTEST_ASSERT_XRDST( MessageUtils::SendMessage( url, msg, handler, params, 0 ) );
+  XRootDStatus st1 = MessageUtils::WaitForResponse( handler, response );
+  delete handler;
+  GTEST_ASSERT_XRDST_NOTOK( st1, errRedirect );
+  EXPECT_TRUE( !response );
+  delete response;
+}
+
+//------------------------------------------------------------------------------
+// Read test
+//------------------------------------------------------------------------------
+void FileTest::ReadTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+  std::string localDataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+  EXPECT_TRUE( testEnv->GetString( "LocalDataPath", localDataPath ) );
+
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string filePath = dataPath + "/cb4aacf1-6f28-42f2-b68a-90a73460f424.dat";
+  std::string fileUrl = address + "/";
+  fileUrl += filePath;
+  localDataPath = realpath(localDataPath.c_str(), NULL);
+  // using the file protocol to access local files, so that we can use XRootD's own functions
+  std::string localFileUrl = "file://localhost" + localDataPath + "/srv1" + filePath;
+
+  //----------------------------------------------------------------------------
+  // Fetch some data and checksum
+  //----------------------------------------------------------------------------
+  const uint32_t MB = 1024*1024;
+  // buffers containing remote file data
+  char *buffer1 = new char[10*MB];
+  char *buffer2 = new char[10*MB];
+  // comparison buffers containing local file data
+  char *buffer1Comp = new char[10*MB];
+  char *buffer2Comp = new char[10*MB];
+  uint32_t bytesRead1 = 0, bytesRead2 = 0;
+  File f, fLocal;
+  StatInfo *statInfo;
+
+  //----------------------------------------------------------------------------
+  // Open the file
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.Open( fileUrl, OpenFlags::Read ) );
+  GTEST_ASSERT_XRDST( fLocal.Open( localFileUrl, OpenFlags::Read ) );
+
+  //----------------------------------------------------------------------------
+  // Stat1
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.Stat( false, statInfo ) );
+  EXPECT_TRUE( statInfo );
+
+  struct stat localStatBuf;
+
+  std::string localFilePath = localDataPath + "/srv1" + filePath;
+
+  int rc = stat(localFilePath.c_str(), &localStatBuf);
+  EXPECT_EQ( rc, 0 );
+  uint64_t fileSize = localStatBuf.st_size;
+  EXPECT_EQ( statInfo->GetSize(), fileSize );
+  EXPECT_TRUE( statInfo->TestFlags( StatInfo::IsReadable ) );
+
+  delete statInfo;
+  statInfo = 0;
+
+  //----------------------------------------------------------------------------
+  // Stat2
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.Stat( true, statInfo ) );
+  EXPECT_TRUE( statInfo );
+  EXPECT_EQ( statInfo->GetSize(), fileSize );
+  EXPECT_TRUE( statInfo->TestFlags( StatInfo::IsReadable ) );
+  delete statInfo;
+
+  //----------------------------------------------------------------------------
+  // Read test
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.Read( 5*MB, 5*MB, buffer1, bytesRead1 ) );
+  GTEST_ASSERT_XRDST( f.Read( fileSize - 3*MB, 4*MB, buffer2, bytesRead2 ) );
+  EXPECT_EQ( bytesRead1, 5*MB );
+  EXPECT_EQ( bytesRead2, 3*MB );
+
+  uint32_t crc = XrdClTests::Utils::ComputeCRC32( buffer1, 5*MB );
+
+  // reinitializing the file cursor
+  bytesRead1 = bytesRead2 = 0;
+  GTEST_ASSERT_XRDST( fLocal.Read( 5*MB, 5*MB, buffer1Comp, bytesRead1 ) );
+  GTEST_ASSERT_XRDST( fLocal.Read( fileSize - 3*MB, 4*MB, buffer2Comp, bytesRead2 ) );
+
+  // compute and compare local file buffer crc
+  uint32_t crcComp = XrdClTests::Utils::ComputeCRC32( buffer1Comp, 5*MB );
+
+  EXPECT_EQ( crc, crcComp );
+
+  // do the same for the second buffer
+  crc = XrdClTests::Utils::ComputeCRC32( buffer2, 3*MB );
+  crcComp = XrdClTests::Utils::ComputeCRC32( buffer2Comp, 3*MB );
+
+  EXPECT_EQ( crc, crcComp );
+
+  // cleanup after ourselves
+  delete [] buffer1;
+  delete [] buffer2;
+  delete [] buffer1Comp;
+  delete [] buffer2Comp;
+
+  GTEST_ASSERT_XRDST( f.Close() );
+  GTEST_ASSERT_XRDST( fLocal.Close() );
+
+  //----------------------------------------------------------------------------
+  // Read ZIP archive test (uncompressed)
+  //----------------------------------------------------------------------------
+  std::string archiveUrl = address + "/" + dataPath + "/data.zip";
+
+  ZipArchive zip;
+  GTEST_ASSERT_XRDST( WaitFor( OpenArchive( zip, archiveUrl, OpenFlags::Read ) ) );
+
+  //----------------------------------------------------------------------------
+  // There are 3 files in the data.zip archive:
+  //  - athena.log
+  //  - paper.txt
+  //  - EastAsianWidth.txt
+  //----------------------------------------------------------------------------
+
+  struct
+  {
+      std::string file;        // file name
+      uint64_t    offset;      // offset in the file
+      uint32_t    size;        // number of characters to be read
+      char        buffer[100]; // the buffer
+      std::string expected;    // expected result
+  } testset[] =
+  {
+      { "athena.log",         65530, 99, {0}, "D__Jet" }, // reads past the end of the file (there are just 6 characters to read not 99)
+      { "paper.txt",          1024,  65, {0}, "igh rate (the order of 100 kHz), the data are usually distributed" },
+      { "EastAsianWidth.txt", 2048,  18, {0}, "8;Na # DIGIT EIGHT" }
+  };
+
+  for( int i = 0; i < 3; ++i )
+  {
+    std::string result;
+    GTEST_ASSERT_XRDST( WaitFor(
+        ReadFrom( zip, testset[i].file, testset[i].offset, testset[i].size, testset[i].buffer ) >>
+          [&result]( auto& s, auto& c )
+          {
+            if( s.IsOK() )
+              result.assign( static_cast(c.buffer), c.length );
+          }
+      ) );
+    EXPECT_EQ( testset[i].expected, result );
+  }
+
+  GTEST_ASSERT_XRDST( WaitFor( CloseArchive( zip ) ) );
+}
+
+
+//------------------------------------------------------------------------------
+// Read test
+//------------------------------------------------------------------------------
+void FileTest::WriteTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string filePath = dataPath + "/testFile.dat";
+  std::string fileUrl = address + "/";
+  fileUrl += filePath;
+
+  //----------------------------------------------------------------------------
+  // Fetch some data and checksum
+  //----------------------------------------------------------------------------
+  const uint32_t MB = 1024*1024;
+  char *buffer1 = new char[4*MB];
+  char *buffer2 = new char[4*MB];
+  char *buffer3 = new char[4*MB];
+  char *buffer4 = new char[4*MB];
+  uint32_t bytesRead1 = 0;
+  uint32_t bytesRead2 = 0;
+  File f1, f2;
+
+  EXPECT_EQ( XrdClTests::Utils::GetRandomBytes( buffer1, 4*MB ), 4*MB );
+  EXPECT_EQ( XrdClTests::Utils::GetRandomBytes( buffer2, 4*MB ), 4*MB );
+  uint32_t crc1 = XrdClTests::Utils::ComputeCRC32( buffer1, 4*MB );
+  crc1 = XrdClTests::Utils::UpdateCRC32( crc1, buffer2, 4*MB );
+
+  //----------------------------------------------------------------------------
+  // Write the data
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f1.Open( fileUrl, OpenFlags::Delete | OpenFlags::Update,
+                           Access::UR | Access::UW ));
+
+  EXPECT_TRUE( f1.Write( 0, 4*MB, buffer1 ).IsOK() );
+  EXPECT_TRUE( f1.Write( 4*MB, 4*MB, buffer2 ).IsOK() );
+  EXPECT_TRUE( f1.Sync().IsOK() );
+  EXPECT_TRUE( f1.Close().IsOK() );
+
+  //----------------------------------------------------------------------------
+  // Read the data and verify the checksums
+  //----------------------------------------------------------------------------
+  StatInfo *statInfo = 0;
+  EXPECT_TRUE( f2.Open( fileUrl, OpenFlags::Read ).IsOK() );
+  EXPECT_TRUE( f2.Stat( false, statInfo ).IsOK() );
+  EXPECT_TRUE( statInfo );
+  EXPECT_EQ( statInfo->GetSize(), 8*MB );
+  EXPECT_TRUE( f2.Read( 0, 4*MB, buffer3, bytesRead1 ).IsOK() );
+  EXPECT_TRUE( f2.Read( 4*MB, 4*MB, buffer4, bytesRead2 ).IsOK() );
+  EXPECT_EQ( bytesRead1, 4*MB );
+  EXPECT_EQ( bytesRead2, 4*MB );
+  uint32_t crc2 = XrdClTests::Utils::ComputeCRC32( buffer3, 4*MB );
+  crc2 = XrdClTests::Utils::UpdateCRC32( crc2, buffer4, 4*MB );
+  EXPECT_TRUE( f2.Close().IsOK() );
+  EXPECT_EQ( crc1, crc2 );
+
+  //----------------------------------------------------------------------------
+  // Truncate test
+  //----------------------------------------------------------------------------
+  EXPECT_TRUE( f1.Open( fileUrl, OpenFlags::Delete | OpenFlags::Update,
+                           Access::UR | Access::UW ).IsOK() );
+  EXPECT_TRUE( f1.Truncate( 20*MB ).IsOK() );
+  EXPECT_TRUE( f1.Close().IsOK() );
+  FileSystem fs( url );
+  StatInfo *response = 0;
+  EXPECT_TRUE( fs.Stat( filePath, response ).IsOK() );
+  EXPECT_TRUE( response );
+  EXPECT_EQ( response->GetSize(), 20*MB );
+  EXPECT_TRUE( fs.Rm( filePath ).IsOK() );
+  delete [] buffer1;
+  delete [] buffer2;
+  delete [] buffer3;
+  delete [] buffer4;
+  delete response;
+  delete statInfo;
+}
+
+//------------------------------------------------------------------------------
+// WriteV test
+//------------------------------------------------------------------------------
+void FileTest::WriteVTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string filePath = dataPath + "/testFile.dat";
+  std::string fileUrl = address + "/";
+  fileUrl += filePath;
+
+  //----------------------------------------------------------------------------
+  // Fetch some data and checksum
+  //----------------------------------------------------------------------------
+  const uint32_t MB = 1024*1024;
+  char *buffer1 = new char[4*MB];
+  char *buffer2 = new char[4*MB];
+  char *buffer3 = new char[8*MB];
+  uint32_t bytesRead1 = 0;
+  File f1, f2;
+
+  EXPECT_EQ( XrdClTests::Utils::GetRandomBytes( buffer1, 4*MB ), 4*MB );
+  EXPECT_EQ( XrdClTests::Utils::GetRandomBytes( buffer2, 4*MB ), 4*MB );
+  uint32_t crc1 = XrdClTests::Utils::ComputeCRC32( buffer1, 4*MB );
+  crc1 = XrdClTests::Utils::UpdateCRC32( crc1, buffer2, 4*MB );
+
+  //----------------------------------------------------------------------------
+  // Prepare IO vector
+  //----------------------------------------------------------------------------
+  int iovcnt = 2;
+  iovec iov[iovcnt];
+  iov[0].iov_base = buffer1;
+  iov[0].iov_len  = 4*MB;
+  iov[1].iov_base = buffer2;
+  iov[1].iov_len  = 4*MB;
+
+  //----------------------------------------------------------------------------
+  // Write the data
+  //----------------------------------------------------------------------------
+  EXPECT_TRUE( f1.Open( fileUrl, OpenFlags::Delete | OpenFlags::Update,
+                           Access::UR | Access::UW ).IsOK() );
+  EXPECT_TRUE( f1.WriteV( 0, iov, iovcnt ).IsOK() );
+  EXPECT_TRUE( f1.Sync().IsOK() );
+  EXPECT_TRUE( f1.Close().IsOK() );
+
+  //----------------------------------------------------------------------------
+  // Read the data and verify the checksums
+  //----------------------------------------------------------------------------
+  StatInfo *statInfo = 0;
+  EXPECT_TRUE( f2.Open( fileUrl, OpenFlags::Read ).IsOK() );
+  EXPECT_TRUE( f2.Stat( false, statInfo ).IsOK() );
+  EXPECT_TRUE( statInfo );
+  EXPECT_EQ( statInfo->GetSize(), 8*MB );
+  EXPECT_TRUE( f2.Read( 0, 8*MB, buffer3, bytesRead1 ).IsOK() );
+  EXPECT_EQ( bytesRead1, 8*MB );
+
+  uint32_t crc2 = XrdClTests::Utils::ComputeCRC32( buffer3, 8*MB );
+  EXPECT_TRUE( f2.Close().IsOK() );
+  EXPECT_EQ( crc1, crc2 );
+
+  FileSystem fs( url );
+  EXPECT_TRUE( fs.Rm( filePath ).IsOK() );
+  delete [] buffer1;
+  delete [] buffer2;
+  delete [] buffer3;
+  delete statInfo;
+}
+
+//------------------------------------------------------------------------------
+// Vector read test
+//------------------------------------------------------------------------------
+void FileTest::VectorReadTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+  std::string localDataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+  EXPECT_TRUE( testEnv->GetString( "LocalDataPath", localDataPath ) );
+
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string filePath = dataPath + "/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat";
+  std::string fileUrl = address + "/";
+  fileUrl += filePath;
+  localDataPath += "/srv1" + filePath;
+  localDataPath = realpath(localDataPath.c_str(), NULL);
+
+  //----------------------------------------------------------------------------
+  // Fetch some data and checksum
+  //----------------------------------------------------------------------------
+  const uint32_t MB = 1024*1024;
+  char *buffer1 = new char[10*MB];
+  char *buffer2 = new char[10*256000];
+  char *buffer1Comp = new char[10*MB];
+  char *buffer2Comp = new char[10*256000];
+  File f, fLocal;
+
+  //----------------------------------------------------------------------------
+  // Build the chunk list
+  //----------------------------------------------------------------------------
+  ChunkList chunkList1;
+  ChunkList chunkList2;
+  for( int i = 0; i < 10; ++i )
+  {
+    chunkList1.push_back( ChunkInfo( (i+1)*1*MB, 1*MB ) );
+    chunkList2.push_back( ChunkInfo( (i+1)*1*MB, 256000 ) );
+  }
+
+  //----------------------------------------------------------------------------
+  // Open the file
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.Open( fileUrl, OpenFlags::Read ) );
+  GTEST_ASSERT_XRDST( fLocal.Open( localDataPath, OpenFlags::Read ) );
+
+  // remote file vread1
+  VectorReadInfo *info = 0;
+  GTEST_ASSERT_XRDST( f.VectorRead( chunkList1, buffer1, info ) );
+  EXPECT_EQ( info->GetSize(), 10*MB );
+  delete info;
+
+  // local file vread1
+  info = 0;
+  GTEST_ASSERT_XRDST( fLocal.VectorRead( chunkList1, buffer1Comp, info ) );
+  EXPECT_EQ( info->GetSize(), 10*MB );
+  delete info;
+
+  // checksum comparison
+  uint32_t crc = 0, crcComp = 0;
+  crc = XrdClTests::Utils::ComputeCRC32( buffer1, 10*MB );
+  crcComp = XrdClTests::Utils::ComputeCRC32( buffer1Comp, 10*MB );
+  EXPECT_EQ( crc, crcComp );
+
+  // remote vread2
+  info = 0;
+  GTEST_ASSERT_XRDST( f.VectorRead( chunkList2, buffer2, info ) );
+  EXPECT_EQ( info->GetSize(), 10*256000 );
+  delete info;
+
+  // local vread2
+  info = 0;
+  GTEST_ASSERT_XRDST( f.VectorRead( chunkList2, buffer2Comp, info ) );
+  EXPECT_EQ( info->GetSize(), 10*256000 );
+  delete info;
+
+  // checksum comparison again
+  crc = XrdClTests::Utils::ComputeCRC32( buffer2, 10*256000 );
+  crcComp = XrdClTests::Utils::ComputeCRC32( buffer2Comp, 10*256000 );
+  EXPECT_EQ( crc, crcComp );
+
+  // cleanup
+  GTEST_ASSERT_XRDST( f.Close() );
+  GTEST_ASSERT_XRDST( fLocal.Close() );
+  delete [] buffer1;
+  delete [] buffer2;
+  delete [] buffer1Comp;
+  delete [] buffer2Comp;
+}
+
+void gen_random_str(char *s, const int len)
+{
+    static const char alphanum[] =
+        "0123456789"
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        "abcdefghijklmnopqrstuvwxyz";
+
+    for (int i = 0; i < len - 1; ++i)
+    {
+        s[i] = alphanum[rand() % (sizeof(alphanum) - 1)];
+    }
+
+    s[len - 1] = 0;
+}
+
+//------------------------------------------------------------------------------
+// Vector write test
+//------------------------------------------------------------------------------
+void FileTest::VectorWriteTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string filePath = dataPath + "/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat";
+  std::string fileUrl = address + "/";
+  fileUrl += filePath;
+
+  //----------------------------------------------------------------------------
+  // Build a random chunk list for vector write
+  //----------------------------------------------------------------------------
+
+  const uint32_t MB = 1024*1024;
+  const uint32_t limit = 10*MB;
+
+  time_t seed = time( 0 );
+  srand( seed );
+  DefaultEnv::GetLog()->Info( UtilityMsg,
+      "Carrying out the VectorWrite test with seed: %d", seed );
+
+  // figure out how many chunks are we going to write/read
+  size_t nbChunks = rand() % 100 + 1;
+
+  XrdCl::ChunkList chunks;
+  size_t   min_offset = 0;
+  uint32_t expectedCrc32 = 0;
+  size_t   totalSize = 0;
+
+  for( size_t i = 0; i < nbChunks; ++i )
+  {
+    // figure out the offset
+    size_t offset = min_offset + rand() % ( limit - min_offset + 1 );
+
+    // figure out the size
+    size_t size = MB + rand() % ( MB + 1 );
+    if( offset + size >= limit )
+      size = limit - offset;
+
+    // generate random string of given size
+    char *buffer = new char[size];
+    gen_random_str( buffer, size );
+
+    // calculate expected checksum
+    expectedCrc32 = XrdClTests::Utils::UpdateCRC32( expectedCrc32, buffer, size );
+    totalSize += size;
+    chunks.push_back( XrdCl::ChunkInfo( offset, size, buffer ) );
+
+    min_offset = offset + size;
+    if( min_offset >= limit )
+      break;
+  }
+
+  //----------------------------------------------------------------------------
+  // Open the file
+  //----------------------------------------------------------------------------
+  File f;
+  GTEST_ASSERT_XRDST( f.Open( fileUrl, OpenFlags::Update ) );
+
+  //----------------------------------------------------------------------------
+  // First do a VectorRead so we can revert to the original state
+  //----------------------------------------------------------------------------
+  char *buffer1 = new char[totalSize];
+  VectorReadInfo *info1 = 0;
+  GTEST_ASSERT_XRDST( f.VectorRead( chunks, buffer1, info1 ) );
+
+  //----------------------------------------------------------------------------
+  // Then do the VectorWrite
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.VectorWrite( chunks ) );
+
+  //----------------------------------------------------------------------------
+  // Now do a vector read and verify that the checksum is the same
+  //----------------------------------------------------------------------------
+  char *buffer2 = new char[totalSize];
+  VectorReadInfo *info2 = 0;
+  GTEST_ASSERT_XRDST( f.VectorRead( chunks, buffer2, info2 ) );
+
+  EXPECT_EQ( info2->GetSize(), totalSize );
+  uint32_t crc32 = XrdClTests::Utils::ComputeCRC32( buffer2, totalSize );
+  EXPECT_EQ( crc32, expectedCrc32 );
+
+  //----------------------------------------------------------------------------
+  // And finally revert to the original state
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.VectorWrite( info1->GetChunks() ) );
+  GTEST_ASSERT_XRDST( f.Close() );
+
+  delete info1;
+  delete info2;
+  delete [] buffer1;
+  delete [] buffer2;
+  for( auto itr = chunks.begin(); itr != chunks.end(); ++itr )
+    delete[] (char*)itr->buffer;
+}
+
+void FileTest::VirtualRedirectorTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string mlUrl1 = address + "/" + dataPath + "/metalink/mlFileTest1.meta4";
+  std::string mlUrl2 = address + "/" + dataPath + "/metalink/mlFileTest2.meta4";
+  std::string mlUrl3 = address + "/" + dataPath + "/metalink/mlFileTest3.meta4";
+  std::string mlUrl4 = address + "/" + dataPath + "/metalink/mlFileTest4.meta4";
+
+  File f1, f2, f3, f4;
+
+  std::string srv1Hostname;
+  EXPECT_TRUE( testEnv->GetString( "Server1URL", srv1Hostname ) );
+  const std::string fileUrl = "root://" + srv1Hostname + "/" + dataPath + "/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat";
+
+  const std::string key = "LastURL";
+  std::string value;
+
+  //----------------------------------------------------------------------------
+  // Open the 1st metalink file
+  // (the metalink contains just one file with a correct location)
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f1.Open( mlUrl1, OpenFlags::Read ) );
+  EXPECT_TRUE( f1.GetProperty( key, value ) );
+  URL lastUrl( value );
+  EXPECT_EQ( lastUrl.GetLocation(), fileUrl );
+  GTEST_ASSERT_XRDST( f1.Close() );
+
+  //----------------------------------------------------------------------------
+  // Open the 2nd metalink file
+  // (the metalink contains 2 files, the one with higher priority does not exist)
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f2.Open( mlUrl2, OpenFlags::Read ) );
+  EXPECT_TRUE( f2.GetProperty( key, value ) );
+  URL lastUrl2( value );
+  EXPECT_EQ( lastUrl2.GetLocation(), fileUrl );
+  GTEST_ASSERT_XRDST( f2.Close() );
+
+  //----------------------------------------------------------------------------
+  // Open the 3rd metalink file
+  // (the metalink contains 2 files, both don't exist)
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST_NOTOK( f3.Open( mlUrl3, OpenFlags::Read ), errErrorResponse );
+
+  //----------------------------------------------------------------------------
+  // Open the 4th metalink file
+  // (the metalink contains 2 files, both exist)
+  //----------------------------------------------------------------------------
+
+  std::string srv3Hostname;
+  EXPECT_TRUE( testEnv->GetString( "Server3URL", srv3Hostname ) );
+  std::string replica1 = "root://" + srv3Hostname + "/" + dataPath + "/3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat";
+
+  std::string srv2Hostname;
+  EXPECT_TRUE( testEnv->GetString( "Server2URL", srv2Hostname ) );
+  const std::string replica2 = "root://" + srv2Hostname + "/" + dataPath + "/3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat";
+
+  GTEST_ASSERT_XRDST( f4.Open( mlUrl4, OpenFlags::Read ) );
+  EXPECT_TRUE( f4.GetProperty( key, value ) );
+  URL lastUrl3( value );
+  EXPECT_EQ( lastUrl3.GetLocation(), replica1 );
+  GTEST_ASSERT_XRDST( f4.Close() );
+  //----------------------------------------------------------------------------
+  // Delete the replica that has been selected by the virtual redirector
+  //----------------------------------------------------------------------------
+  FileSystem fs( replica1 );
+  GTEST_ASSERT_XRDST( fs.Rm( dataPath + "/3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat" ) );
+  //----------------------------------------------------------------------------
+  // Now reopen the file
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f4.Open( mlUrl4, OpenFlags::Read ) );
+  EXPECT_TRUE( f4.GetProperty( key, value ) );
+  URL lastUrl4( value );
+  EXPECT_EQ( lastUrl4.GetLocation(), replica2 );
+  GTEST_ASSERT_XRDST( f4.Close() );
+  //----------------------------------------------------------------------------
+  // Recreate the deleted file
+  //----------------------------------------------------------------------------
+  CopyProcess  process;
+  PropertyList properties, results;
+  properties.Set( "source",       replica2 );
+  properties.Set( "target",       replica1 );
+  GTEST_ASSERT_XRDST( process.AddJob( properties, &results ) );
+  GTEST_ASSERT_XRDST( process.Prepare() );
+  GTEST_ASSERT_XRDST( process.Run(0) );
+}
+
+void FileTest::XAttrTest()
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Get the environment variables
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "DiskServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  std::string filePath = dataPath + "/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat";
+  std::string fileUrl = address + "/" + filePath;
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+
+  File file;
+  GTEST_ASSERT_XRDST( file.Open( fileUrl, OpenFlags::Update ) );
+
+  std::map attributes
+  {
+      std::make_pair( "version",  "v1.2.3-45" ),
+      std::make_pair( "checksum", "2ccc0e85556a6cd193dd8d2b40aab50c" ),
+      std::make_pair( "index",    "4" )
+  };
+
+  //----------------------------------------------------------------------------
+  // Test SetXAttr
+  //----------------------------------------------------------------------------
+  std::vector attrs;
+  auto itr1 = attributes.begin();
+  for( ; itr1 != attributes.end() ; ++itr1 )
+    attrs.push_back( std::make_tuple( itr1->first, itr1->second ) );
+
+  std::vector result1;
+  GTEST_ASSERT_XRDST( file.SetXAttr( attrs, result1 ) );
+
+  auto itr2 = result1.begin();
+  for( ; itr2 != result1.end() ; ++itr2 )
+    GTEST_ASSERT_XRDST( itr2->status );
+
+  //----------------------------------------------------------------------------
+  // Test GetXAttr
+  //----------------------------------------------------------------------------
+  std::vector names;
+  itr1 = attributes.begin();
+  for( ; itr1 != attributes.end() ; ++itr1 )
+    names.push_back( itr1->first );
+
+  std::vector result2;
+  GTEST_ASSERT_XRDST( file.GetXAttr( names, result2 ) );
+
+  auto itr3 = result2.begin();
+  for( ; itr3 != result2.end() ; ++itr3 )
+  {
+    GTEST_ASSERT_XRDST( itr3->status );
+    auto match = attributes.find( itr3->name );
+    EXPECT_NE( match, attributes.end() );
+    EXPECT_EQ( match->second, itr3->value );
+  }
+
+  //----------------------------------------------------------------------------
+  // Test ListXAttr
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( file.ListXAttr( result2 ) );
+
+  itr3 = result2.begin();
+  for( ; itr3 != result2.end() ; ++itr3 )
+  {
+    GTEST_ASSERT_XRDST( itr3->status );
+    auto match = attributes.find( itr3->name );
+    EXPECT_NE( match, attributes.end() );
+    EXPECT_EQ( match->second, itr3->value );
+  }
+
+  //----------------------------------------------------------------------------
+  // Test DelXAttr
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( file.DelXAttr( names, result1 ) );
+
+  itr2 = result1.begin();
+  for( ; itr2 != result1.end() ; ++itr2 )
+    GTEST_ASSERT_XRDST( itr2->status );
+
+  GTEST_ASSERT_XRDST( file.Close() );
+}
diff --git a/tests/XrdCl/XrdClLocalFileHandlerTest.cc b/tests/XrdCl/XrdClLocalFileHandlerTest.cc
new file mode 100644
index 00000000000..cbf02eae078
--- /dev/null
+++ b/tests/XrdCl/XrdClLocalFileHandlerTest.cc
@@ -0,0 +1,548 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+#include "TestEnv.hh"
+#include "GTestXrdHelpers.hh"
+#include "XrdCl/XrdClFile.hh"
+#include "XrdSys/XrdSysPlatform.hh"
+
+#include 
+#include 
+#include 
+#include 
+
+using namespace XrdClTests;
+
+//------------------------------------------------------------------------------
+// Declaration
+//------------------------------------------------------------------------------
+class LocalFileHandlerTest: public ::testing::Test
+{
+  public:
+    void SetUp() override;
+    void TearDown() override;
+
+    void CreateTestFileFunc( std::string url, std::string content = "GenericTestFile" );
+    void readTestFunc( bool offsetRead, uint32_t offset );
+    void OpenCloseTest();
+    void ReadTest();
+    void ReadWithOffsetTest();
+    void WriteTest();
+    void WriteWithOffsetTest();
+    void WriteMkdirTest();
+    void TruncateTest();
+    void VectorReadTest();
+    void VectorWriteTest();
+    void SyncTest();
+    void WriteVTest();
+    void XAttrTest();
+
+    std::string m_tmpdir;
+};
+
+void LocalFileHandlerTest::SetUp()
+{
+  char cpath[MAXPATHLEN];
+  ASSERT_TRUE(getcwd(cpath, sizeof(cpath))) <<
+    "Could not get current working directory";
+  m_tmpdir = std::string(cpath) + "/tmp";
+}
+
+void LocalFileHandlerTest::TearDown()
+{
+  /* empty */
+}
+
+//----------------------------------------------------------------------------
+// Create the file to be tested
+//----------------------------------------------------------------------------
+void LocalFileHandlerTest::CreateTestFileFunc( std::string url, std::string content ){
+   errno = 0;
+   mode_t openmode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
+   int fd = open( url.c_str(), O_RDWR | O_CREAT | O_TRUNC, openmode );
+   EXPECT_NE( fd, -1 );
+   EXPECT_EQ( errno, 0 );
+   int rc = write( fd, content.c_str(), content.size() );
+   EXPECT_EQ( rc, int( content.size() ) );
+   EXPECT_EQ( errno, 0 );
+   rc = close( fd );
+   EXPECT_EQ( rc, 0 );
+}
+
+//----------------------------------------------------------------------------
+// Performs a ReadTest
+//----------------------------------------------------------------------------
+void LocalFileHandlerTest::readTestFunc(bool offsetRead, uint32_t offset){
+   using namespace XrdCl;
+   std::string targetURL = m_tmpdir + "/lfilehandlertestfileread";
+   std::string toBeWritten = "tenBytes10";
+   std::string expectedRead = "Byte";
+   uint32_t size =
+      ( offsetRead ? expectedRead.size() : toBeWritten.size() );
+   char *buffer = new char[size];
+   uint32_t bytesRead = 0;
+
+   //----------------------------------------------------------------------------
+   // Write file with POSIX calls to ensure correct write
+   //----------------------------------------------------------------------------
+   CreateTestFileFunc( targetURL, toBeWritten );
+
+   //----------------------------------------------------------------------------
+   // Open and Read File
+   //----------------------------------------------------------------------------
+   OpenFlags::Flags flags = OpenFlags::Update;
+   Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR;
+   File *file = new File();
+   GTEST_ASSERT_XRDST( file->Open( targetURL, flags, mode ) );
+   EXPECT_TRUE( file->IsOpen() );
+   GTEST_ASSERT_XRDST( file->Read( offset, size, buffer, bytesRead ) );
+   GTEST_ASSERT_XRDST( file->Close() );
+
+   std::string read( buffer, size );
+   if (offsetRead) EXPECT_EQ( expectedRead, read );
+   else EXPECT_EQ( toBeWritten, read );
+
+   EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+
+   delete[] buffer;
+   delete file;
+
+}
+
+TEST_F(LocalFileHandlerTest, SyncTest){
+   using namespace XrdCl;
+   std::string targetURL = m_tmpdir + "/lfilehandlertestfilesync";
+   CreateTestFileFunc( targetURL );
+
+   //----------------------------------------------------------------------------
+   // Open and Sync File
+   //----------------------------------------------------------------------------
+   OpenFlags::Flags flags = OpenFlags::Update;
+   Access::Mode mode = Access::UR | Access::UW | Access::GR | Access::OR;
+   File *file = new File();
+   GTEST_ASSERT_XRDST( file->Open( targetURL, flags, mode ) );
+   GTEST_ASSERT_XRDST( file->Sync() );
+   GTEST_ASSERT_XRDST( file->Close() );
+   EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+   delete file;
+}
+
+TEST_F(LocalFileHandlerTest, OpenCloseTest){
+   using namespace XrdCl;
+   std::string targetURL = m_tmpdir + "/lfilehandlertestfileopenclose";
+   CreateTestFileFunc( targetURL );
+
+   //----------------------------------------------------------------------------
+   // Open existing file
+   //----------------------------------------------------------------------------
+   OpenFlags::Flags flags = OpenFlags::Update;
+   Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR;
+   File *file = new File();
+   GTEST_ASSERT_XRDST( file->Open( targetURL, flags, mode ) );
+   EXPECT_TRUE( file->IsOpen() );
+   GTEST_ASSERT_XRDST( file->Close() );
+   EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+
+   //----------------------------------------------------------------------------
+   // Try open non-existing file
+   //----------------------------------------------------------------------------
+   EXPECT_TRUE( !file->Open( targetURL, flags, mode ).IsOK() );
+   EXPECT_TRUE( !file->IsOpen() );
+
+   //----------------------------------------------------------------------------
+   // Try close non-opened file, return has to be error
+   //----------------------------------------------------------------------------
+   EXPECT_EQ( file->Close().status, stError );
+   delete file;
+}
+
+TEST_F(LocalFileHandlerTest, WriteTest){
+   using namespace XrdCl;
+   std::string targetURL = m_tmpdir + "/lfilehandlertestfilewrite";
+   std::string toBeWritten = "tenBytes1\0";
+   uint32_t writeSize = toBeWritten.size();
+   CreateTestFileFunc( targetURL, "" );
+   char *buffer = new char[writeSize];
+   //----------------------------------------------------------------------------
+   // Open and Write File
+   //----------------------------------------------------------------------------
+   OpenFlags::Flags flags = OpenFlags::Update;
+   Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR;
+   File *file = new File();
+   GTEST_ASSERT_XRDST( file->Open( targetURL, flags, mode ) );
+   EXPECT_TRUE( file->IsOpen() );
+   GTEST_ASSERT_XRDST( file->Write( 0, writeSize, toBeWritten.c_str()) );
+   GTEST_ASSERT_XRDST( file->Close() );
+
+   //----------------------------------------------------------------------------
+   // Read file with POSIX calls to confirm correct write
+   //----------------------------------------------------------------------------
+   int fd = open( targetURL.c_str(), O_RDWR );
+   int rc = read( fd, buffer, int( writeSize ) );
+   EXPECT_EQ( rc, int( writeSize ) );
+   std::string read( (char *)buffer, writeSize );
+   EXPECT_EQ( toBeWritten, read );
+   EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+   delete[] buffer;
+   delete file;
+}
+
+TEST_F(LocalFileHandlerTest, WriteWithOffsetTest){
+   using namespace XrdCl;
+   std::string targetURL = m_tmpdir + "/lfilehandlertestfilewriteoffset";
+   std::string toBeWritten = "tenBytes10";
+   std::string notToBeOverwritten = "front";
+   uint32_t writeSize = toBeWritten.size();
+   uint32_t offset = notToBeOverwritten.size();
+   void *buffer = new char[offset];
+   CreateTestFileFunc( targetURL, notToBeOverwritten );
+
+   //----------------------------------------------------------------------------
+   // Open and Write File
+   //----------------------------------------------------------------------------
+   OpenFlags::Flags flags = OpenFlags::Update;
+   Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR;
+   File *file = new File();
+   GTEST_ASSERT_XRDST( file->Open( targetURL, flags, mode ) );
+   EXPECT_TRUE( file->IsOpen() );
+   GTEST_ASSERT_XRDST( file->Write( offset, writeSize, toBeWritten.c_str()) );
+   GTEST_ASSERT_XRDST( file->Close() );
+
+   //----------------------------------------------------------------------------
+   // Read file with POSIX calls to confirm correct write
+   //----------------------------------------------------------------------------
+   int fd = open( targetURL.c_str(), O_RDWR );
+   int rc = read( fd, buffer, offset );
+   EXPECT_EQ( rc, int( offset ) );
+   std::string read( (char *)buffer, offset );
+   EXPECT_EQ( notToBeOverwritten, read );
+   EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+   delete[] (char*)buffer;
+   delete file;
+}
+
+TEST_F(LocalFileHandlerTest, WriteMkdirTest){
+   using namespace XrdCl;
+   std::string targetURL = m_tmpdir + "/testdir/further/muchfurther/evenfurther/lfilehandlertestfilewrite";
+   std::string toBeWritten = "tenBytes10";
+   uint32_t writeSize = toBeWritten.size();
+   char *buffer = new char[writeSize];
+
+   //----------------------------------------------------------------------------
+   // Open and Write File
+   //----------------------------------------------------------------------------
+   OpenFlags::Flags flags = OpenFlags::Update | OpenFlags::MakePath | OpenFlags::New;
+   Access::Mode mode = Access::UR|Access::UW|Access::UX;
+   File *file = new File();
+   GTEST_ASSERT_XRDST( file->Open( targetURL, flags, mode ) );
+   EXPECT_TRUE( file->IsOpen() );
+   GTEST_ASSERT_XRDST( file->Write( 0, writeSize, toBeWritten.c_str()) );
+   GTEST_ASSERT_XRDST( file->Close() );
+
+   //----------------------------------------------------------------------------
+   // Read file with POSIX calls to confirm correct write
+   //----------------------------------------------------------------------------
+   int fd = open( targetURL.c_str(), O_RDWR );
+   int rc = read( fd, buffer, writeSize );
+   EXPECT_EQ( rc, int( writeSize ) );
+   std::string read( buffer, writeSize );
+   EXPECT_EQ( toBeWritten, read );
+   EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+   delete[] buffer;
+   delete file;
+}
+
+TEST_F(LocalFileHandlerTest, ReadTests){
+   // Normal read test
+   readTestFunc(false, 0);
+   // Read with offset test
+   readTestFunc(true, 3);
+}
+
+TEST_F(LocalFileHandlerTest, TruncateTest){
+   using namespace XrdCl;
+   //----------------------------------------------------------------------------
+   // Initialize
+   //----------------------------------------------------------------------------
+   std::string targetURL = m_tmpdir + "/lfilehandlertestfiletruncate";
+
+   CreateTestFileFunc(targetURL);
+   //----------------------------------------------------------------------------
+   // Prepare truncate
+   //----------------------------------------------------------------------------
+   OpenFlags::Flags flags = OpenFlags::Update | OpenFlags::Force;
+   Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR;
+   File *file = new File();
+   uint32_t bytesRead = 0;
+   uint32_t truncateSize = 5;
+
+   //----------------------------------------------------------------------------
+   // Read after truncate, but with greater length. bytesRead must still be
+   // truncate size if truncate works as intended
+   //----------------------------------------------------------------------------
+   GTEST_ASSERT_XRDST( file->Open( targetURL, flags, mode ) );
+   GTEST_ASSERT_XRDST( file->Truncate( truncateSize ) );
+   char *buffer = new char[truncateSize + 3];
+   GTEST_ASSERT_XRDST( file->Read( 0, truncateSize + 3, buffer, bytesRead ) );
+   EXPECT_EQ( truncateSize, bytesRead );
+   GTEST_ASSERT_XRDST( file->Close() );
+   EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+   delete file;
+   delete[] buffer;
+}
+
+TEST_F(LocalFileHandlerTest, VectorReadTest)
+{
+   using namespace XrdCl;
+
+   //----------------------------------------------------------------------------
+   // Initialize
+   //----------------------------------------------------------------------------
+   std::string targetURL = m_tmpdir + "/lfilehandlertestfilevectorread";
+   CreateTestFileFunc( targetURL );
+   VectorReadInfo *info = 0;
+   ChunkList chunks;
+
+   //----------------------------------------------------------------------------
+   // Prepare VectorRead
+   //----------------------------------------------------------------------------
+   OpenFlags::Flags flags = OpenFlags::Update;
+   Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR;
+   File file;
+   GTEST_ASSERT_XRDST( file.Open( targetURL, flags, mode ) );
+
+   //----------------------------------------------------------------------------
+   // VectorRead no cursor
+   //----------------------------------------------------------------------------
+
+   chunks.push_back( ChunkInfo( 0, 5, new char[5] ) );
+   chunks.push_back( ChunkInfo( 10, 5, new char[5] ) );
+   GTEST_ASSERT_XRDST( file.VectorRead( chunks, NULL, info ) );
+   GTEST_ASSERT_XRDST( file.Close() );
+   EXPECT_EQ( info->GetSize(), 10 );
+   EXPECT_EQ( 0, memcmp( "Gener",
+                                    info->GetChunks()[0].buffer,
+                                    info->GetChunks()[0].length ) );
+   EXPECT_EQ( 0, memcmp( "tFile",
+                                    info->GetChunks()[1].buffer,
+                                    info->GetChunks()[1].length ) );
+   delete[] (char*)chunks[0].buffer;
+   delete[] (char*)chunks[1].buffer;
+   delete info;
+
+   //----------------------------------------------------------------------------
+   // VectorRead cursor
+   //----------------------------------------------------------------------------
+   char *buffer = new char[10];
+   chunks.clear();
+   chunks.push_back( ChunkInfo( 0, 5, 0 ) );
+   chunks.push_back( ChunkInfo( 10, 5, 0 ) );
+   info = 0;
+
+   GTEST_ASSERT_XRDST( file.Open( targetURL, flags, mode ) );
+   GTEST_ASSERT_XRDST( file.VectorRead( chunks, buffer, info ) );
+   GTEST_ASSERT_XRDST( file.Close() );
+   EXPECT_EQ( info->GetSize(), 10 );
+   EXPECT_EQ( 0, memcmp( "GenertFile",
+                                    info->GetChunks()[0].buffer,
+                                    info->GetChunks()[0].length ) );
+   EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+
+   delete[] buffer;
+   delete info;
+}
+
+TEST_F(LocalFileHandlerTest, VectorWriteTest)
+{
+   using namespace XrdCl;
+
+   //----------------------------------------------------------------------------
+   // Initialize
+   //----------------------------------------------------------------------------
+   std::string targetURL = m_tmpdir + "/lfilehandlertestfilevectorwrite";
+   CreateTestFileFunc( targetURL );
+   ChunkList chunks;
+
+   //----------------------------------------------------------------------------
+   // Prepare VectorWrite
+   //----------------------------------------------------------------------------
+   OpenFlags::Flags flags = OpenFlags::Update;
+   Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR;
+   File file;
+   GTEST_ASSERT_XRDST( file.Open( targetURL, flags, mode ) );
+
+   //----------------------------------------------------------------------------
+   // VectorWrite
+   //----------------------------------------------------------------------------
+
+   ChunkInfo chunk( 0, 5, new char[5] );
+   memset( chunk.buffer, 'A', chunk.length );
+   chunks.push_back( chunk );
+   chunk = ChunkInfo( 10, 5, new char[5] );
+   memset( chunk.buffer, 'B', chunk.length );
+   chunks.push_back( chunk );
+
+   GTEST_ASSERT_XRDST( file.VectorWrite( chunks ) );
+
+   //----------------------------------------------------------------------------
+   // Verify with VectorRead
+   //----------------------------------------------------------------------------
+
+   VectorReadInfo *info = 0;
+   char *buffer = new char[10];
+   GTEST_ASSERT_XRDST( file.VectorRead( chunks, buffer, info ) );
+
+   EXPECT_EQ( 0, memcmp( buffer, "AAAAABBBBB", 10 ) );
+
+   GTEST_ASSERT_XRDST( file.Close() );
+   EXPECT_EQ( info->GetSize(), 10 );
+
+   delete[] (char*)chunks[0].buffer;
+   delete[] (char*)chunks[1].buffer;
+   delete[] buffer;
+   delete   info;
+
+   EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+}
+
+TEST_F(LocalFileHandlerTest, WriteVTest)
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  std::string targetURL = m_tmpdir + "/lfilehandlertestfilewritev";
+  CreateTestFileFunc( targetURL );
+
+  //----------------------------------------------------------------------------
+  // Prepare WriteV
+  //----------------------------------------------------------------------------
+  OpenFlags::Flags flags = OpenFlags::Update;
+  Access::Mode mode = Access::UR|Access::UW|Access::GR|Access::OR;
+  File file;
+  GTEST_ASSERT_XRDST( file.Open( targetURL, flags, mode ) );
+
+  char str[] = "WriteVTest";
+  std::vector buffer( 10 );
+  std::copy( str, str + sizeof( str ) - 1, buffer.begin() );
+  int iovcnt = 2;
+  iovec iov[iovcnt];
+  iov[0].iov_base = buffer.data();
+  iov[0].iov_len  = 6;
+  iov[1].iov_base = buffer.data() + 6;
+  iov[1].iov_len  = 4;
+  GTEST_ASSERT_XRDST( file.WriteV( 7, iov, iovcnt ) );
+
+  uint32_t bytesRead = 0;
+  buffer.resize( 17 );
+  GTEST_ASSERT_XRDST( file.Read( 0, 17, buffer.data(), bytesRead ) );
+  EXPECT_EQ( buffer.size(), 17 );
+  std::string expected = "GenericWriteVTest";
+  EXPECT_EQ( std::string( buffer.data(), buffer.size() ), expected );
+  GTEST_ASSERT_XRDST( file.Close() );
+  EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+}
+
+TEST_F(LocalFileHandlerTest, XAttrTest)
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  // (we do the test in /data as /tmp might be on tpmfs,
+  //  which does not support xattrs)
+  // In this case, /data is /data inside of the build directory
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+  std::string localDataPath;
+  EXPECT_TRUE( testEnv->GetString( "LocalDataPath", localDataPath ) );
+
+  char resolved_path[MAXPATHLEN];
+  localDataPath = realpath(localDataPath.c_str(), resolved_path);
+
+  std::string targetURL = localDataPath + "/metaman/lfilehandlertestfilexattr";
+  CreateTestFileFunc( targetURL );
+
+  File f;
+  GTEST_ASSERT_XRDST( f.Open( targetURL, OpenFlags::Update ) );
+
+  //----------------------------------------------------------------------------
+  // Test XAttr Set
+  //----------------------------------------------------------------------------
+  std::vector attrs;
+  attrs.push_back( xattr_t( "version", "v3.3.3" ) );
+  attrs.push_back( xattr_t( "description", "a very important file" ) );
+  attrs.push_back( xattr_t( "checksum", "0x22334455" ) );
+
+  std::vector st_resp;
+
+  GTEST_ASSERT_XRDST( f.SetXAttr( attrs, st_resp ) );
+
+  std::vector::iterator itr1;
+  for( itr1 = st_resp.begin(); itr1 != st_resp.end(); ++itr1 )
+    GTEST_ASSERT_XRDST( itr1->status );
+
+  //----------------------------------------------------------------------------
+  // Test XAttr Get
+  //----------------------------------------------------------------------------
+  std::vector names;
+  names.push_back( "version" );
+  names.push_back( "description" );
+  std::vector resp;
+  GTEST_ASSERT_XRDST( f.GetXAttr( names, resp ) );
+
+  GTEST_ASSERT_XRDST( resp[0].status );
+  GTEST_ASSERT_XRDST( resp[1].status );
+
+  EXPECT_EQ( resp.size(), 2 );
+  int vid = resp[0].name == "version" ? 0 : 1;
+  int did = vid == 0 ? 1 : 0;
+  EXPECT_EQ( resp[vid].name, std::string("version") );
+  EXPECT_EQ( resp[vid].value, std::string("v3.3.3") );
+  EXPECT_EQ( resp[did].name, std::string("description") );
+  EXPECT_EQ( resp[did].value, std::string("a very important file") );
+
+  //----------------------------------------------------------------------------
+  // Test XAttr Del
+  //----------------------------------------------------------------------------
+  names.clear();
+  names.push_back( "description" );
+  st_resp.clear();
+  GTEST_ASSERT_XRDST( f.DelXAttr( names, st_resp ) );
+  EXPECT_EQ( st_resp.size(), 1 );
+  GTEST_ASSERT_XRDST( st_resp[0].status );
+
+  //----------------------------------------------------------------------------
+  // Test XAttr List
+  //----------------------------------------------------------------------------
+  resp.clear();
+  GTEST_ASSERT_XRDST( f.ListXAttr( resp ) );
+  EXPECT_EQ( resp.size(), 2 );
+  vid = resp[0].name == "version" ? 0 : 1;
+  int cid = vid == 0 ? 1 : 0;
+  EXPECT_EQ( resp[vid].name, std::string("version") );
+  EXPECT_EQ( resp[vid].value, std::string("v3.3.3") );
+  EXPECT_EQ( resp[cid].name, std::string("checksum") );
+  EXPECT_EQ( resp[cid].value, std::string("0x22334455") );
+
+  //----------------------------------------------------------------------------
+  // Cleanup
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.Close() );
+  EXPECT_EQ( remove( targetURL.c_str() ), 0 );
+}
diff --git a/tests/XrdCl/XrdClOperationsWorkflowTest.cc b/tests/XrdCl/XrdClOperationsWorkflowTest.cc
new file mode 100644
index 00000000000..02041c9fbdc
--- /dev/null
+++ b/tests/XrdCl/XrdClOperationsWorkflowTest.cc
@@ -0,0 +1,989 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// This file is part of the XRootD software suite.
+//
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//
+// In applying this licence, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+//------------------------------------------------------------------------------
+
+#include "TestEnv.hh"
+#include "IdentityPlugIn.hh"
+#include "GTestXrdHelpers.hh"
+#include "XrdCl/XrdClURL.hh"
+#include "XrdCl/XrdClOperations.hh"
+#include "XrdCl/XrdClParallelOperation.hh"
+#include "XrdCl/XrdClFileOperations.hh"
+#include "XrdCl/XrdClFileSystemOperations.hh"
+#include "XrdCl/XrdClCheckpointOperation.hh"
+#include "XrdCl/XrdClFwd.hh"
+
+#include 
+#include 
+
+using namespace XrdClTests;
+
+//------------------------------------------------------------------------------
+// Declaration
+//------------------------------------------------------------------------------
+class WorkflowTest: public ::testing::Test
+{
+  public:
+    void ReadingWorkflowTest();
+    void WritingWorkflowTest();
+    void MissingParameterTest();
+    void OperationFailureTest();
+    void DoubleRunningTest();
+    void ParallelTest();
+    void FileSystemWorkflowTest();
+    void MixedWorkflowTest();
+    void WorkflowWithFutureTest();
+    void XAttrWorkflowTest();
+    void MkDirAsyncTest();
+    void CheckpointTest();
+};
+
+namespace {
+  using namespace XrdCl;
+
+  XrdCl::URL GetAddress(){
+      Env *testEnv = TestEnv::GetEnv();
+      std::string address;
+      EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+      return XrdCl::URL(address);
+  }
+
+  std::string GetPath(const std::string &fileName){
+      Env *testEnv = TestEnv::GetEnv();
+
+      std::string dataPath;
+      EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+      return dataPath + "/" + fileName;
+  }
+
+
+  std::string GetFileUrl(const std::string &fileName){
+      Env *testEnv = TestEnv::GetEnv();
+
+      std::string address;
+      std::string dataPath;
+
+      EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+      EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+      URL url( address );
+      EXPECT_TRUE( url.IsValid() );
+
+      std::string path = dataPath + "/" + fileName;
+      std::string fileUrl = address + "/" + path;
+
+      return fileUrl;
+  }
+
+  class TestingHandler: public ResponseHandler {
+      public:
+          TestingHandler(){
+              executed = false;
+          }
+
+          void HandleResponseWithHosts(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList) {
+              delete hostList;
+              HandleResponse(status, response);
+          }
+
+          void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) {
+              GTEST_ASSERT_XRDST(*status);
+              delete status;
+              delete response;
+              executed = true;
+          }
+
+          bool Executed(){
+              return executed;
+          }
+
+      protected:
+          bool executed;
+  };
+
+  class ExpectErrorHandler: public ResponseHandler
+  {
+    public:
+        ExpectErrorHandler(){
+            executed = false;
+        }
+
+        void HandleResponseWithHosts(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response, XrdCl::HostList *hostList) {
+            delete hostList;
+            HandleResponse(status, response);
+        }
+
+        void HandleResponse(XrdCl::XRootDStatus *status, XrdCl::AnyObject *response) {
+            EXPECT_TRUE( !status->IsOK() );
+            delete status;
+            delete response;
+            executed = true;
+        }
+
+        bool Executed(){
+            return executed;
+        }
+
+    protected:
+        bool executed;
+  };
+}
+
+
+TEST(WorkflowTest, ReadingWorkflowTest){
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  std::string fileUrl = GetFileUrl("cb4aacf1-6f28-42f2-b68a-90a73460f424.dat");
+  File f;
+
+  //----------------------------------------------------------------------------
+  // Create handlers
+  //----------------------------------------------------------------------------
+  TestingHandler openHandler;
+  TestingHandler readHandler;
+  TestingHandler closeHandler;
+
+  //----------------------------------------------------------------------------
+  // Forward parameters between operations
+  //----------------------------------------------------------------------------
+  Fwd size;
+  Fwd    buffer;
+
+  //----------------------------------------------------------------------------
+  // Create and execute workflow
+  //----------------------------------------------------------------------------
+
+  const OpenFlags::Flags flags = OpenFlags::Read;
+  uint64_t offset = 0;
+
+  Env *testEnv = TestEnv::GetEnv();
+  std::string localDataPath;
+  std::string dataPath;
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+  EXPECT_TRUE( testEnv->GetString( "LocalDataPath", localDataPath ) );
+  std::string filePath = dataPath + "/cb4aacf1-6f28-42f2-b68a-90a73460f424.dat";
+
+  struct stat localStatBuf;
+  std::string localFilePath = localDataPath + "/srv1" + filePath;
+  int rc = stat(localFilePath.c_str(), &localStatBuf);
+  EXPECT_EQ( rc, 0 );
+  uint64_t fileSize = localStatBuf.st_size;
+
+  auto &&pipe = Open( f, fileUrl, flags ) >> openHandler // by reference
+              | Stat( f, true) >> [fileSize, size, buffer]( XRootDStatus &status, StatInfo &stat) mutable
+                  {
+                    GTEST_ASSERT_XRDST( status );
+                    EXPECT_EQ( stat.GetSize(), fileSize );
+                    size = stat.GetSize();
+                    buffer = new char[stat.GetSize()];
+                  }
+              | Read( f, offset, size, buffer ) >> &readHandler // by pointer
+              | Close( f ) >> closeHandler; // by reference
+
+  XRootDStatus status = WaitFor( pipe );
+  GTEST_ASSERT_XRDST( status );
+
+  EXPECT_TRUE( openHandler.Executed() );
+  EXPECT_TRUE( readHandler.Executed() );
+  EXPECT_TRUE( closeHandler.Executed() );
+
+  delete[] reinterpret_cast( *buffer );
+}
+
+
+TEST(WorkflowTest, WritingWorkflowTest){
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  std::string fileUrl = GetFileUrl("testFile.dat");
+  auto flags = OpenFlags::Write | OpenFlags::Delete | OpenFlags::Update;
+  std::string texts[3] = {"First line\n", "Second line\n", "Third line\n"};
+  File f;
+
+  auto url = GetAddress();
+  FileSystem fs(url);
+  auto relativePath = GetPath("testFile.dat");
+
+  auto createdFileSize = texts[0].size() + texts[1].size() + texts[2].size();
+
+  //----------------------------------------------------------------------------
+  // Create handlers
+  //----------------------------------------------------------------------------
+  std::packaged_task parser {
+    []( XRootDStatus& status, ChunkInfo &chunk )
+      {
+        GTEST_ASSERT_XRDST( status );
+        char* buffer = reinterpret_cast( chunk.buffer );
+        std::string ret( buffer, chunk.length );
+        delete[] buffer;
+        return ret;
+      }
+  };
+  std::future rdresp = parser.get_future();
+
+  //----------------------------------------------------------------------------
+  // Forward parameters between operations
+  //----------------------------------------------------------------------------
+  Fwd> iov;
+  Fwd           size;
+  Fwd              buffer;
+
+  //----------------------------------------------------------------------------
+  // Create and execute workflow
+  //----------------------------------------------------------------------------
+  Pipeline pipe = Open( f, fileUrl, flags ) >> [iov, texts]( XRootDStatus &status ) mutable
+                    {
+                      GTEST_ASSERT_XRDST( status );
+                      std::vector vec( 3 );
+                      vec[0].iov_base = strdup( texts[0].c_str() );
+                      vec[0].iov_len  = texts[0].size();
+                      vec[1].iov_base = strdup( texts[1].c_str() );
+                      vec[1].iov_len  = texts[1].size();
+                      vec[2].iov_base = strdup( texts[2].c_str() );
+                      vec[2].iov_len  = texts[2].size();
+                      iov = std::move( vec );
+                    }
+                | WriteV( f, 0, iov )
+                | Sync( f )
+                | Stat( f, true ) >> [size, buffer, createdFileSize]( XRootDStatus &status, StatInfo &info ) mutable
+                    {
+                      GTEST_ASSERT_XRDST( status );
+                      EXPECT_EQ( createdFileSize, info.GetSize() );
+                      size   = info.GetSize();
+                      buffer = new char[info.GetSize()];
+                    }
+                | Read( f, 0, size, buffer ) >> parser
+                | Close( f )
+                | Rm( fs, relativePath );
+
+  XRootDStatus status = WaitFor( std::move( pipe ) );
+  GTEST_ASSERT_XRDST( status );
+  EXPECT_EQ( rdresp.get(), texts[0] + texts[1] + texts[2] );
+
+  free( (*iov)[0].iov_base );
+  free( (*iov)[1].iov_base );
+  free( (*iov)[2].iov_base );
+}
+
+
+TEST(WorkflowTest, MissingParameterTest){
+    using namespace XrdCl;
+
+    //----------------------------------------------------------------------------
+    // Initialize
+    //----------------------------------------------------------------------------
+    std::string fileUrl = GetFileUrl("cb4aacf1-6f28-42f2-b68a-90a73460f424.dat");
+    File f;
+
+    //----------------------------------------------------------------------------
+    // Create handlers
+    //----------------------------------------------------------------------------
+    ExpectErrorHandler readHandler;
+
+    //----------------------------------------------------------------------------
+    // Bad forwards
+    //----------------------------------------------------------------------------
+    Fwd size;
+    Fwd    buffer;
+
+    //----------------------------------------------------------------------------
+    // Create and execute workflow
+    //----------------------------------------------------------------------------
+
+    bool error = false, closed = false;
+    const OpenFlags::Flags flags = OpenFlags::Read;
+    uint64_t offset = 0;
+
+    Pipeline pipe = Open( f, fileUrl, flags )
+                  | Stat( f, true )
+                  | Read( f, offset, size, buffer ) >> readHandler // by reference
+                  | Close( f ) >> [&]( XRootDStatus& st )
+                      {
+                        closed = true;
+                      }
+                  | Final( [&]( const XRootDStatus& st )
+                      {
+                        error = !st.IsOK();
+                      });
+
+    XRootDStatus status = WaitFor( std::move( pipe ) );
+    EXPECT_TRUE( status.IsError() );
+    //----------------------------------------------------------------------------
+    // If there is an error, last handlers should not be executed
+    //----------------------------------------------------------------------------
+    EXPECT_TRUE( readHandler.Executed() );
+    EXPECT_TRUE( !closed & error );
+}
+
+
+
+TEST(WorkflowTest, OperationFailureTest){
+    using namespace XrdCl;
+
+    //----------------------------------------------------------------------------
+    // Initialize
+    //----------------------------------------------------------------------------
+    std::string fileUrl = GetFileUrl("noexisting.dat");
+    File f;
+
+    //----------------------------------------------------------------------------
+    // Create handlers
+    //----------------------------------------------------------------------------
+    ExpectErrorHandler openHandler;
+    std::future statresp;
+    std::future readresp;
+    std::future closeresp;
+
+    //----------------------------------------------------------------------------
+    // Create and execute workflow
+    //----------------------------------------------------------------------------
+
+    const OpenFlags::Flags flags = OpenFlags::Read;
+    auto &&pipe = Open( f, fileUrl, flags ) >> &openHandler // by pointer
+                | Stat( f, true ) >> statresp
+                | Read( f, 0, 0, nullptr ) >> readresp
+                | Close( f ) >> closeresp;
+
+    XRootDStatus status = WaitFor( pipe ); // by obscure operation type
+    EXPECT_TRUE( status.IsError() );
+
+    //----------------------------------------------------------------------------
+    // If there is an error, handlers should not be executed
+    //----------------------------------------------------------------------------
+    EXPECT_TRUE(openHandler.Executed());
+
+    try
+    {
+      statresp.get();
+    }
+    catch( PipelineException &ex )
+    {
+      GTEST_ASSERT_XRDST_NOTOK( ex.GetError(), errPipelineFailed );
+    }
+
+    try
+    {
+      readresp.get();
+    }
+    catch( PipelineException &ex )
+    {
+      GTEST_ASSERT_XRDST_NOTOK( ex.GetError(), errPipelineFailed );
+    }
+
+    try
+    {
+      closeresp.get();
+    }
+    catch( PipelineException &ex )
+    {
+      GTEST_ASSERT_XRDST_NOTOK( ex.GetError(), errPipelineFailed );
+    }
+}
+
+
+TEST(WorkflowTest, DoubleRunningTest){
+    using namespace XrdCl;
+
+    //----------------------------------------------------------------------------
+    // Initialize
+    //----------------------------------------------------------------------------
+    std::string fileUrl = GetFileUrl("cb4aacf1-6f28-42f2-b68a-90a73460f424.dat");
+    File f;
+
+    //----------------------------------------------------------------------------
+    // Create and execute workflow
+    //----------------------------------------------------------------------------
+
+    const OpenFlags::Flags flags = OpenFlags::Read;
+    bool opened = false, closed = false;
+
+    auto &&pipe = Open( f, fileUrl, flags ) >> [&]( XRootDStatus &status ){ opened = status.IsOK(); }
+                | Close( f ) >> [&]( XRootDStatus &status ){ closed = status.IsOK(); };
+
+    std::future ftr = Async( pipe );
+
+    //----------------------------------------------------------------------------
+    // Running workflow again should fail
+    //----------------------------------------------------------------------------
+    try
+    {
+        Async( pipe );
+        EXPECT_TRUE( false );
+    }
+    catch( std::logic_error &err )
+    {
+
+    }
+
+
+    XRootDStatus status = ftr.get();
+
+    //----------------------------------------------------------------------------
+    // Running workflow again should fail
+    //----------------------------------------------------------------------------
+    try
+    {
+        Async( pipe );
+        EXPECT_TRUE( false );
+    }
+    catch( std::logic_error &err )
+    {
+
+    }
+
+    EXPECT_TRUE( status.IsOK() );
+
+    EXPECT_TRUE( opened );
+    EXPECT_TRUE( closed );
+}
+
+
+TEST(WorkflowTest, ParallelTest){
+    using namespace XrdCl;
+
+    //----------------------------------------------------------------------------
+    // Initialize
+    //----------------------------------------------------------------------------
+    File lockFile;
+    File firstFile;
+    File secondFile;
+
+    std::string lockFileName = "lockfile.lock";
+    std::string dataFileName = "testFile.dat";
+
+    std::string lockUrl = GetFileUrl(lockFileName);
+    std::string firstFileUrl = GetFileUrl("cb4aacf1-6f28-42f2-b68a-90a73460f424.dat");
+    std::string secondFileUrl = GetFileUrl(dataFileName);
+
+    const auto readFlags = OpenFlags::Read;
+    const auto createFlags = OpenFlags::Delete;
+
+    // ----------------------------------------------------------------------------
+    // Create lock file and new data file
+    // ----------------------------------------------------------------------------
+    auto f = new File();
+    auto dataF = new File();
+
+    std::vector pipes; pipes.reserve( 2 );
+    pipes.emplace_back( Open( f, lockUrl, createFlags ) | Close( f ) );
+    pipes.emplace_back( Open( dataF, secondFileUrl, createFlags ) | Close( dataF ) );
+    GTEST_ASSERT_XRDST( WaitFor( Parallel( pipes ) >> []( XRootDStatus &status ){ GTEST_ASSERT_XRDST( status ); } ) );
+    EXPECT_TRUE( pipes.empty() );
+
+    delete f;
+    delete dataF;
+
+    //----------------------------------------------------------------------------
+    // Create and execute workflow
+    //----------------------------------------------------------------------------
+    uint64_t offset = 0;
+    uint32_t size = 50  ;
+    char* firstBuffer  = new char[size]();
+    char* secondBuffer = new char[size]();
+
+    Fwd url1, url2;
+
+    bool lockHandlerExecuted = false;
+    auto lockOpenHandler = [&,url1, url2]( XRootDStatus &status ) mutable
+        {
+          url1 = GetFileUrl( "cb4aacf1-6f28-42f2-b68a-90a73460f424.dat" );
+          url2 = GetFileUrl( dataFileName );
+          lockHandlerExecuted = true;
+        };
+
+    std::future parallelresp, closeresp;
+
+    Pipeline firstPipe = Open( firstFile, url1, readFlags )
+                       | Read( firstFile, offset, size, firstBuffer )
+                       | Close( firstFile );
+
+    Pipeline secondPipe = Open( secondFile, url2, readFlags )
+                        | Read( secondFile, offset, size, secondBuffer )
+                        | Close( secondFile );
+
+    Pipeline pipe = Open( lockFile, lockUrl, readFlags ) >> lockOpenHandler
+                  | Parallel( firstPipe, secondPipe ) >> parallelresp
+                  | Close( lockFile ) >> closeresp;
+
+    XRootDStatus status = WaitFor( std::move( pipe ) );
+    EXPECT_TRUE(status.IsOK());
+
+    EXPECT_TRUE(lockHandlerExecuted);
+
+    try
+    {
+      parallelresp.get();
+      closeresp.get();
+    }
+    catch( std::exception &ex )
+    {
+      EXPECT_TRUE( false );
+    }
+
+    delete[] firstBuffer;
+    delete[] secondBuffer;
+
+    //----------------------------------------------------------------------------
+    // Remove lock file and data file
+    //----------------------------------------------------------------------------
+    f = new File();
+    dataF = new File();
+
+    auto url = GetAddress();
+    FileSystem fs( url );
+
+    auto lockRelativePath = GetPath(lockFileName);
+    auto dataRelativePath = GetPath(dataFileName);
+
+    bool exec1 = false, exec2 = false;
+    Pipeline deletingPipe( Parallel( Rm( fs, lockRelativePath ) >> [&]( XRootDStatus &status ){ GTEST_ASSERT_XRDST( status ); exec1 = true; },
+                                     Rm( fs, dataRelativePath ) >> [&]( XRootDStatus &status ){ GTEST_ASSERT_XRDST( status ); exec2 = true; } ) );
+    GTEST_ASSERT_XRDST( WaitFor( std::move( deletingPipe ) ) );
+
+    EXPECT_TRUE( exec1 );
+    EXPECT_TRUE( exec2 );
+
+    delete f;
+    delete dataF;
+
+    //----------------------------------------------------------------------------
+    // Test the policies
+    //----------------------------------------------------------------------------
+    std::string url_exists = GetPath("1db882c8-8cd6-4df1-941f-ce669bad3458.dat");
+    std::string not_exists = GetPath("blablabla.txt");
+    GTEST_ASSERT_XRDST( WaitFor( Parallel( Stat( fs, url_exists ), Stat( fs, url_exists ) ).Any() ) );
+
+    std::string also_exists = GetPath("3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat");
+    GTEST_ASSERT_XRDST( WaitFor( Parallel( Stat( fs, url_exists ),
+                                             Stat( fs, also_exists ),
+                                             Stat( fs, not_exists ) ).Some( 2 ) ) );
+    std::atomic errcnt( 0 );
+    std::atomic okcnt( 0 );
+
+    auto hndl = [&]( auto s, auto i )
+      {
+        if( s.IsOK() ) ++okcnt;
+        else ++errcnt;
+      };
+
+    GTEST_ASSERT_XRDST( WaitFor( Parallel( Stat( fs, url_exists )  >> hndl,
+                                             Stat( fs, also_exists ) >> hndl,
+                                             Stat( fs, not_exists )  >> hndl ).AtLeast( 1 ) ) );
+    EXPECT_EQ( okcnt, 2 );
+    EXPECT_EQ( errcnt, 1 );
+}
+
+
+TEST(WorkflowTest, FileSystemWorkflowTest){
+    using namespace XrdCl;
+
+    TestingHandler mkDirHandler;
+    TestingHandler locateHandler;
+    TestingHandler moveHandler;
+    TestingHandler secondLocateHandler;
+    TestingHandler removeHandler;
+
+    auto url = GetAddress();
+    FileSystem fs( url );
+
+    std::string newDirUrl = GetPath("sourceDirectory");
+    std::string destDirUrl = GetPath("destDirectory");
+
+    auto noneFlags = OpenFlags::None;
+
+    Pipeline fsPipe = MkDir( fs, newDirUrl, MkDirFlags::None, Access::None ) >> mkDirHandler
+                    | Locate( fs, newDirUrl, noneFlags ) >> locateHandler
+                    | Mv( fs, newDirUrl, destDirUrl ) >> moveHandler
+                    | Locate( fs, destDirUrl, OpenFlags::Refresh ) >> secondLocateHandler
+                    | RmDir( fs, destDirUrl ) >> removeHandler;
+
+    Pipeline pipe( std::move( fsPipe) );
+
+    XRootDStatus status = WaitFor( std::move( pipe ) );
+    EXPECT_TRUE(status.IsOK());
+
+    EXPECT_TRUE(mkDirHandler.Executed());
+    EXPECT_TRUE(locateHandler.Executed());
+    EXPECT_TRUE(moveHandler.Executed());
+    EXPECT_TRUE(secondLocateHandler.Executed());
+    EXPECT_TRUE(removeHandler.Executed());
+}
+
+
+TEST(WorkflowTest, MixedWorkflowTest){
+    using namespace XrdCl;
+
+    const size_t nbFiles = 2;
+
+    FileSystem fs( GetAddress() );
+    File file[nbFiles];
+
+    auto flags = OpenFlags::Write | OpenFlags::Delete | OpenFlags::Update;
+
+    std::string dirName = "tempDir";
+    std::string dirPath = GetPath( dirName );
+
+    std::string firstFileName = dirName + "/firstFile";
+    std::string secondFileName = dirName + "/secondFile";
+    std::string url[nbFiles] = { GetFileUrl(firstFileName), GetFileUrl(secondFileName) };
+
+    std::string path[nbFiles] = { GetPath(firstFileName), GetPath(secondFileName) };
+
+    std::string content[nbFiles] = { "First file content",  "Second file content" };
+    char* text[nbFiles] = { const_cast(content[0].c_str()), const_cast(content[1].c_str()) };
+    size_t length[nbFiles] = { content[0].size(), content[1].size() };
+
+
+    Fwd size[nbFiles];
+    Fwd buffer[nbFiles];
+
+    std::future ftr[nbFiles];
+
+    bool cleaningHandlerExecuted = false;
+    auto cleaningHandler = [&](XRootDStatus &status, LocationInfo& info)
+      {
+        LocationInfo::Iterator it;
+        for( it = info.Begin(); it != info.End(); ++it )
+        {
+            auto url = URL(it->GetAddress());
+            FileSystem fs(url);
+            auto st = fs.RmDir(dirPath);
+            EXPECT_TRUE(st.IsOK());
+        }
+        cleaningHandlerExecuted = true;
+      };
+
+    std::vector fileWorkflows;
+    for( size_t i = 0; i < nbFiles; ++i )
+    {
+      auto &&operation = Open( file[i], url[i], flags )
+                       | Write( file[i], 0, length[i], text[i] )
+                       | Sync( file[i] )
+                       | Stat( file[i], true ) >> [size, buffer, i]( XRootDStatus &status, StatInfo &info ) mutable
+                           {
+                             GTEST_ASSERT_XRDST( status );
+                             size[i] = info.GetSize();
+                             buffer[i] = new char[*size[i]];
+                           }
+                       | Read( file[i], 0, size[i], buffer[i] ) >> ftr[i]
+                       | Close( file[i] );
+      fileWorkflows.emplace_back( operation );
+    }
+
+    Pipeline pipe = MkDir( fs, dirPath, MkDirFlags::None, Access::None ) >> []( XRootDStatus &status ){ GTEST_ASSERT_XRDST( status ); }
+                  | Parallel( fileWorkflows )
+                  | Rm( fs, path[0] )
+                  | Rm( fs, path[1] )
+                  | DeepLocate( fs, dirPath, OpenFlags::Refresh ) >> cleaningHandler;
+
+    XRootDStatus status = WaitFor( std::move( pipe ) );
+    GTEST_ASSERT_XRDST( status );
+
+    for( size_t i = 0; i < nbFiles; ++i )
+    {
+      ChunkInfo chunk = ftr[i].get();
+      char *buffer = reinterpret_cast( chunk.buffer );
+      std::string result( buffer, chunk.length );
+      delete[] buffer;
+      EXPECT_EQ( result, content[i] );
+    }
+
+    EXPECT_TRUE(cleaningHandlerExecuted);
+}
+
+
+TEST(WorkflowTest, WorkflowWithFutureTest)
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string filePath = dataPath + "/cb4aacf1-6f28-42f2-b68a-90a73460f424.dat";
+  std::string fileUrl = address + "/";
+  fileUrl += filePath;
+
+  //----------------------------------------------------------------------------
+  // Fetch some data and checksum
+  //----------------------------------------------------------------------------
+  const uint32_t MB = 1024*1024;
+  char *expected = new char[10*MB];
+  char *buffer   = new char[10*MB];
+  uint32_t bytesRead = 0;
+  File f;
+
+  //----------------------------------------------------------------------------
+  // Open and Read and Close in standard way
+  //----------------------------------------------------------------------------
+  GTEST_ASSERT_XRDST( f.Open( fileUrl, OpenFlags::Read ) );
+  GTEST_ASSERT_XRDST( f.Read( 1*MB, 10*MB, expected, bytesRead ) );
+  EXPECT_EQ( bytesRead, 10*MB );
+  GTEST_ASSERT_XRDST( f.Close() );
+
+  //----------------------------------------------------------------------------
+  // Now do the test
+  //----------------------------------------------------------------------------
+  File file;
+  std::future ftr;
+  Pipeline pipeline = Open( file, fileUrl, OpenFlags::Read ) | Read( file, 1*MB, 10*MB, buffer ) >> ftr | Close( file );
+  std::future status = Async( std::move( pipeline ) );
+
+  try
+  {
+    ChunkInfo result = ftr.get();
+    EXPECT_TRUE( result.length = bytesRead );
+    EXPECT_EQ( strncmp( expected, (char*)result.buffer, bytesRead ), 0 );
+  }
+  catch( PipelineException &ex )
+  {
+    EXPECT_TRUE( false );
+  }
+
+  GTEST_ASSERT_XRDST( status.get() )
+
+  delete[] expected;
+  delete[] buffer;
+}
+
+TEST(WorkflowTest, XAttrWorkflowTest)
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string filePath = dataPath + "/cb4aacf1-6f28-42f2-b68a-90a73460f424.dat";
+  std::string fileUrl = address + "/";
+  fileUrl += filePath;
+
+  //----------------------------------------------------------------------------
+  // Now do the test
+  //----------------------------------------------------------------------------
+  std::string xattr_name  = "xrd.name";
+  std::string xattr_value = "ala ma kota";
+  File file1, file2;
+
+  // set extended attribute
+  Pipeline set = Open( file1, fileUrl, OpenFlags::Write )
+               | SetXAttr( file1, xattr_name, xattr_value )
+               | Close( file1 );
+  GTEST_ASSERT_XRDST( WaitFor( std::move( set ) ) );
+
+  // read and delete the extended attribute
+  std::future rsp1;
+
+  Pipeline get_del = Open( file2, fileUrl, OpenFlags::Update )
+                   | GetXAttr( file2, xattr_name ) >> rsp1
+                   | DelXAttr( file2, xattr_name )
+                   | Close( file2 );
+
+  GTEST_ASSERT_XRDST( WaitFor( std::move( get_del ) ) );
+
+  try
+  {
+    EXPECT_EQ( xattr_value, rsp1.get() );
+  }
+  catch( PipelineException &ex )
+  {
+    EXPECT_TRUE( false );
+  }
+
+  //----------------------------------------------------------------------------
+  // Test the bulk operations
+  //----------------------------------------------------------------------------
+  std::vector names{ "xrd.name1", "xrd.name2" };
+  std::vector attrs;
+  attrs.push_back( xattr_t( names[0], "ala ma kota" ) );
+  attrs.push_back( xattr_t( names[1], "ela nic nie ma" ) );
+  File file3, file4;
+
+  // set extended attributes
+  Pipeline set_bulk = Open( file3, fileUrl, OpenFlags::Write )
+                    | SetXAttr( file3, attrs )
+                    | Close( file3 );
+  GTEST_ASSERT_XRDST( WaitFor( std::move( set_bulk ) ) );
+
+  // read and delete the extended attribute
+  Pipeline get_del_bulk = Open( file4, fileUrl, OpenFlags::Update )
+                        | ListXAttr( file4 ) >>
+                          [&]( XRootDStatus &status, std::vector &rsp )
+                            {
+                              GTEST_ASSERT_XRDST( status );
+                              EXPECT_EQ( rsp.size(), attrs.size() );
+                              for( size_t i = 0; i < rsp.size(); ++i )
+                              {
+                                auto itr = std::find_if( attrs.begin(), attrs.end(),
+                                    [&]( xattr_t &a ){ return std::get<0>( a ) == rsp[i].name; } );
+                                EXPECT_NE( itr, attrs.end() );
+                                EXPECT_EQ( std::get<1>( *itr ), rsp[i].value );
+                              }
+                            }
+                        | DelXAttr( file4, names )
+                        | Close( file4 );
+
+  GTEST_ASSERT_XRDST( WaitFor( std::move( get_del_bulk ) ) );
+
+  //----------------------------------------------------------------------------
+  // Test FileSystem xattr
+  //----------------------------------------------------------------------------
+  FileSystem fs( fileUrl );
+  std::future rsp2;
+
+  Pipeline pipeline = SetXAttr( fs, filePath, xattr_name, xattr_value )
+                    | GetXAttr( fs, filePath, xattr_name ) >> rsp2
+                    | ListXAttr( fs, filePath ) >>
+                      [&]( XRootDStatus &status, std::vector &rsp )
+                        {
+                          GTEST_ASSERT_XRDST( status );
+                          EXPECT_EQ( rsp.size(), 1 );
+                          EXPECT_EQ( rsp[0].name, xattr_name );
+                          EXPECT_EQ( rsp[0].value, xattr_value );
+                        }
+                    | DelXAttr( fs, filePath, xattr_name );
+
+  GTEST_ASSERT_XRDST( WaitFor( std::move( pipeline ) ) );
+
+  try
+  {
+    EXPECT_EQ( xattr_value, rsp2.get() );
+  }
+  catch( PipelineException &ex )
+  {
+    EXPECT_TRUE( false );
+  }
+}
+
+TEST(WorkflowTest, MkDirAsyncTest) {
+  using namespace XrdCl;
+  std::string asyncTestFile = GetPath("MkDirAsyncTest");
+
+  FileSystem fs( GetAddress() );
+
+  std::packaged_task mkdirTask{
+    []( XrdCl::XRootDStatus &st ) {
+        if (!st.IsOK())
+          throw XrdCl::PipelineException( st );
+    }};
+
+  XrdCl::Access::Mode access = XrdCl::Access::Mode::UR | XrdCl::Access::Mode::UW |
+                               XrdCl::Access::Mode::UX | XrdCl::Access::Mode::GR |
+                               XrdCl::Access::Mode::GW | XrdCl::Access::Mode::GX;
+
+  auto &&t = Async( MkDir( fs, asyncTestFile, XrdCl::MkDirFlags::None, access ) >> mkdirTask |
+                    RmDir( fs, asyncTestFile )
+                  );
+
+  EXPECT_EQ(t.get().status, stOK);
+}
+
+TEST(WorkflowTest, CheckpointTest) {
+  using namespace XrdCl;
+
+  File f1;
+  const char data[] = "Murzynek Bambo w Afryce mieszka,\n"
+                      "czarna ma skore ten nasz kolezka\n"
+                      "Uczy sie pilnie przez cale ranki\n"
+                      "Ze swej murzynskiej pierwszej czytanki.";
+
+  std::string chkpttestFile =  GetPath("chkpttest.txt");
+  std::string serverName = (GetAddress()).GetURL();
+  std::string url = serverName + chkpttestFile;
+
+  GTEST_ASSERT_XRDST( WaitFor( Open( f1, url, OpenFlags::New | OpenFlags::Write ) |
+                                 Write( f1, 0, sizeof( data ), data ) |
+                                 Close( f1 ) ) );
+
+  //---------------------------------------------------------------------------
+  // Update the file without commiting the checkpoint
+  //---------------------------------------------------------------------------
+  File f2;
+  const char update[] = "Jan A Kowalski";
+
+  GTEST_ASSERT_XRDST( WaitFor( Open( f2, url, OpenFlags::Update ) |
+                                 Checkpoint( f2, ChkPtCode::BEGIN ) |
+                                 ChkptWrt( f2, 0, sizeof( update ), update ) |
+                                 Close( f2 ) ) );
+
+  File f3;
+  char readout[sizeof( data )];
+  // readout the data to see if the update was succesful (it shouldn't be)
+  GTEST_ASSERT_XRDST( WaitFor( Open( f3, url, OpenFlags::Read ) |
+                                 Read( f3, 0, sizeof( readout ), readout ) |
+                                 Close( f3 ) ) );
+  // we expect the data to be unchanged
+  EXPECT_EQ( strncmp( readout, data, sizeof( data ) ), 0 );
+
+  //---------------------------------------------------------------------------
+  // Update the file and commit the changes
+  //---------------------------------------------------------------------------
+  File f4;
+  GTEST_ASSERT_XRDST( WaitFor( Open( f4, url, OpenFlags::Update ) |
+                                 Checkpoint( f4, ChkPtCode::BEGIN ) |
+                                 ChkptWrt( f4, 0, sizeof( update ), update ) |
+                                 Checkpoint( f4, ChkPtCode::COMMIT ) |
+                                 Close( f4 ) ) );
+  File f5;
+  // readout the data to see if the update was succesful (it shouldn't be)
+  GTEST_ASSERT_XRDST( WaitFor( Open( f5, url, OpenFlags::Read ) |
+                                 Read( f5, 0, sizeof( readout ), readout ) |
+                                 Close( f5 ) ) );
+  // we expect the data to be unchanged
+  EXPECT_EQ( strncmp( readout, update, sizeof( update ) ), 0 );
+  EXPECT_TRUE( strncmp( readout + sizeof( update ), data + sizeof( update ),
+                           sizeof( data ) - sizeof( update ) ) == 0 );
+
+  //---------------------------------------------------------------------------
+  // Now clean up
+  //---------------------------------------------------------------------------
+  FileSystem fs( url );
+  GTEST_ASSERT_XRDST( WaitFor( Rm( fs, chkpttestFile ) ) );
+}
+
diff --git a/tests/XrdCl/XrdClPoller.cc b/tests/XrdCl/XrdClPoller.cc
new file mode 100644
index 00000000000..5569c64799b
--- /dev/null
+++ b/tests/XrdCl/XrdClPoller.cc
@@ -0,0 +1,267 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+#include "XrdCl/XrdClPoller.hh"
+#include "GTestXrdHelpers.hh"
+#include "Server.hh"
+#include "Utils.hh"
+#include "TestEnv.hh"
+#include "XrdCl/XrdClURL.hh"
+#include "XrdCl/XrdClUtils.hh"
+#include "XrdCl/XrdClSocket.hh"
+#include 
+
+#include 
+
+#include "XrdCl/XrdClPollerBuiltIn.hh"
+
+using namespace XrdClTests;
+using namespace testing;
+
+//------------------------------------------------------------------------------
+// Client handler
+//------------------------------------------------------------------------------
+class RandomPumpHandler: public ClientHandler
+{
+  public:
+    //--------------------------------------------------------------------------
+    // Pump some random data through the socket
+    //--------------------------------------------------------------------------
+    virtual void HandleConnection( int socket )
+    {
+      XrdCl::ScopedDescriptor scopetDesc( socket );
+      XrdCl::Log *log = TestEnv::GetLog();
+
+      uint8_t  packets = random() % 100;
+      uint16_t packetSize;
+      char     buffer[50000];
+      log->Debug( 1, "Sending %d packets to the client", packets );
+
+      for( int i = 0; i < packets; ++i )
+      {
+        packetSize = random() % 50000;
+        log->Dump( 1, "Sending %d packet, %d bytes of data", i, packetSize );
+        if( Utils::GetRandomBytes( buffer, packetSize ) != packetSize )
+        {
+          log->Error( 1, "Unable to get %d bytes of random data", packetSize );
+          return;
+        }
+
+        if( ::write( socket, buffer, packetSize ) != packetSize )
+        {
+          log->Error( 1, "Unable to send the %d bytes of random data",
+                      packetSize );
+          return;
+        }
+        UpdateSentData( buffer, packetSize );
+      }
+    }
+};
+
+//------------------------------------------------------------------------------
+// Client handler factory
+//------------------------------------------------------------------------------
+class RandomPumpHandlerFactory: public ClientHandlerFactory
+{
+  public:
+    virtual ClientHandler *CreateHandler()
+    {
+      return new RandomPumpHandler();
+    }
+};
+
+//------------------------------------------------------------------------------
+// Socket listener
+//------------------------------------------------------------------------------
+class SocketHandler: public XrdCl::SocketHandler
+{
+  public:
+    //--------------------------------------------------------------------------
+    // Initializer
+    //--------------------------------------------------------------------------
+    virtual void Initialize( XrdCl::Poller *poller )
+    {
+      pPoller = poller;
+    }
+
+    //--------------------------------------------------------------------------
+    // Handle an event
+    //--------------------------------------------------------------------------
+    virtual void Event( uint8_t type,
+                        XrdCl::Socket *socket )
+    {
+      //------------------------------------------------------------------------
+      // Read event
+      //------------------------------------------------------------------------
+      if( type & ReadyToRead )
+      {
+        char    buffer[50000];
+        int     desc = socket->GetFD();
+        ssize_t ret = 0;
+
+        while( 1 )
+        {
+          char     *current   = buffer;
+          uint32_t  spaceLeft = 50000;
+          while( (spaceLeft > 0) &&
+                 ((ret = ::read( desc, current, spaceLeft )) > 0) )
+          {
+            current   += ret;
+            spaceLeft -= ret;
+          }
+
+          UpdateTransferMap( socket->GetSockName(), buffer, 50000-spaceLeft );
+
+          if( ret == 0 )
+          {
+            pPoller->RemoveSocket( socket );
+            return;
+          }
+
+          if( ret < 0 )
+          {
+            if( errno != EAGAIN && errno != EWOULDBLOCK )
+              pPoller->EnableReadNotification( socket, false );
+            return;
+          }
+        }
+      }
+
+      //------------------------------------------------------------------------
+      // Timeout
+      //------------------------------------------------------------------------
+      if( type & ReadTimeOut )
+        pPoller->RemoveSocket( socket );
+    }
+
+    //--------------------------------------------------------------------------
+    // Update the checksums
+    //--------------------------------------------------------------------------
+    void UpdateTransferMap( const std::string &sockName,
+                            const void        *buffer,
+                            uint32_t           size )
+    {
+      //------------------------------------------------------------------------
+      // Check if we have an entry in the map
+      //------------------------------------------------------------------------
+      std::pair res;
+      Server::TransferMap::iterator it;
+      res = pMap.insert( std::make_pair( sockName, std::make_pair( 0, 0 ) ) );
+      it = res.first;
+      if( res.second == true )
+      {
+        it->second.first  = 0;
+        it->second.second = Utils::ComputeCRC32( 0, 0 );
+      }
+
+      //------------------------------------------------------------------------
+      // Update the entry
+      //------------------------------------------------------------------------
+      it->second.first += size;
+      it->second.second = Utils::UpdateCRC32( it->second.second, buffer, size );
+    }
+
+    //--------------------------------------------------------------------------
+    //! Get the stats of the received data
+    //--------------------------------------------------------------------------
+    std::pair GetReceivedStats(
+                                      const std::string sockName ) const
+    {
+      Server::TransferMap::const_iterator it = pMap.find( sockName );
+      if( it == pMap.end() )
+        return std::make_pair( 0, 0 );
+      return it->second;
+    }
+
+  private:
+    Server::TransferMap  pMap;
+    XrdCl::Poller   *pPoller;
+};
+
+
+//------------------------------------------------------------------------------
+// PollerTest class declaration
+//------------------------------------------------------------------------------
+
+class PollerTest : public ::testing::Test {};
+
+//------------------------------------------------------------------------------
+// Test the functionality the built-in poller
+//------------------------------------------------------------------------------
+
+TEST(PollerTest, FunctionTest)
+{
+  XrdCl::Poller *poller = new XrdCl::PollerBuiltIn(); // only uses built-in poller
+
+  using XrdCl::Socket;
+  using XrdCl::URL;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Server server( Server::Both );
+  Socket s[3];
+  EXPECT_TRUE( server.Setup( 9999, 3, new RandomPumpHandlerFactory() ) );
+  EXPECT_TRUE( server.Start() );
+  EXPECT_TRUE( poller->Initialize() );
+  EXPECT_TRUE( poller->Start() );
+
+  //----------------------------------------------------------------------------
+  // Connect the sockets
+  //----------------------------------------------------------------------------
+  SocketHandler *handler = new SocketHandler();
+  for( int i = 0; i < 3; ++i )
+  {
+    GTEST_ASSERT_XRDST( s[i].Initialize() );
+    GTEST_ASSERT_XRDST( s[i].Connect( "localhost", 9999 ) );
+    EXPECT_TRUE( poller->AddSocket( &s[i], handler ) );
+    EXPECT_TRUE( poller->EnableReadNotification( &s[i], true, 60 ) );
+    EXPECT_TRUE( poller->IsRegistered( &s[i] ) );
+  }
+
+  //----------------------------------------------------------------------------
+  // All the business happens elsewhere so we have nothing better to do
+  // here that wait, otherwise server->stop will hang.
+  //----------------------------------------------------------------------------
+  ::sleep(1);
+
+  //----------------------------------------------------------------------------
+  // Cleanup
+  //----------------------------------------------------------------------------
+  EXPECT_TRUE( poller->Stop() );
+  EXPECT_TRUE( server.Stop() );
+  EXPECT_TRUE( poller->Finalize() );
+
+  std::pair stats[3];
+  std::pair statsServ[3];
+  for( int i = 0; i < 3; ++i )
+  {
+    EXPECT_TRUE( !poller->IsRegistered( &s[i] ) );
+    stats[i] = handler->GetReceivedStats( s[i].GetSockName() );
+    statsServ[i] = server.GetSentStats( s[i].GetSockName() );
+    EXPECT_EQ( stats[i].first, statsServ[i].first );
+    EXPECT_EQ( stats[i].second, statsServ[i].second );
+  }
+
+  for( int i = 0; i < 3; ++i )
+    s[i].Close();
+
+  delete handler;
+  delete poller;
+}
+
diff --git a/tests/XrdCl/XrdClPostMasterTest.cc b/tests/XrdCl/XrdClPostMasterTest.cc
new file mode 100644
index 00000000000..0e91c6db3cc
--- /dev/null
+++ b/tests/XrdCl/XrdClPostMasterTest.cc
@@ -0,0 +1,575 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#include 
+
+#include "TestEnv.hh"
+#include "GTestXrdHelpers.hh"
+
+using namespace XrdClTests;
+
+//------------------------------------------------------------------------------
+// Declaration
+//------------------------------------------------------------------------------
+class PostMasterTest: public ::testing::Test
+{
+  public:
+    void FunctionalTest();
+    void ThreadingTest();
+    void PingIPv6();
+    void MultiIPConnectionTest();
+};
+
+//------------------------------------------------------------------------------
+// Tear down the post master
+//------------------------------------------------------------------------------
+namespace
+{
+  class PostMasterFetch
+  {
+    public:
+      PostMasterFetch() { }
+      ~PostMasterFetch()  { }
+      XrdCl::PostMaster *Get() {
+        return XrdCl::DefaultEnv::GetPostMaster();
+      }
+      XrdCl::PostMaster *Reset() {
+        XrdCl::PostMaster *pm = Get();
+        pm->Stop();
+        pm->Finalize();
+        EXPECT_NE( pm->Initialize(), 0 );
+        EXPECT_NE( pm->Start(), 0 );
+        return pm;
+      }
+  };
+}
+
+//------------------------------------------------------------------------------
+// Message filter
+//------------------------------------------------------------------------------
+class XrdFilter
+{
+  friend class SyncMsgHandler;
+
+  public:
+    XrdFilter( unsigned char id0 = 0, unsigned char id1 = 0 )
+    {
+      streamId[0] = id0;
+      streamId[1] = id1;
+    }
+
+    virtual bool Filter( const XrdCl::Message *msg )
+    {
+      ServerResponse *resp = (ServerResponse *)msg->GetBuffer();
+      if( resp->hdr.streamid[0] == streamId[0] &&
+          resp->hdr.streamid[1] == streamId[1] )
+        return true;
+      return false;
+    }
+
+    virtual uint16_t GetSid() const
+    {
+      return (((uint16_t)streamId[1] << 8) | (uint16_t)streamId[0]);
+    }
+
+    unsigned char streamId[2];
+};
+
+//------------------------------------------------------------------------------
+// Synchronous Message Handler
+//------------------------------------------------------------------------------
+class SyncMsgHandler : public XrdCl::MsgHandler
+{
+  public:
+    SyncMsgHandler() :
+      sem( 0 ), request( nullptr ), response( nullptr ), expiration( 0 )
+    {
+    }
+
+  private:
+
+    XrdFilter                        filter;
+    XrdSysSemaphore                  sem;
+    XrdCl::XRootDStatus              status;
+    const XrdCl::Message            *request;
+    std::shared_ptr  response;
+    time_t                           expiration;
+
+  public:
+
+    //------------------------------------------------------------------------
+    // Examine an incoming message, and decide on the action to be taken
+    //------------------------------------------------------------------------
+    virtual uint16_t Examine( std::shared_ptr &msg )
+    {
+      if( filter.Filter( msg.get() ) )
+      {
+        response = msg;
+        return RemoveHandler;
+      }
+      return Ignore;
+    }
+
+    //------------------------------------------------------------------------
+    // Reexamine the incoming message, and decide on the action to be taken
+    //------------------------------------------------------------------------
+    virtual uint16_t InspectStatusRsp()
+    {
+      return XrdCl::MsgHandler::Action::None;
+    }
+
+    //------------------------------------------------------------------------
+    // Get handler sid
+    //------------------------------------------------------------------------
+    virtual uint16_t GetSid() const
+    {
+      return filter.GetSid();
+    }
+
+    //------------------------------------------------------------------------
+    // Process the message if it was "taken" by the examine action
+    //------------------------------------------------------------------------
+    virtual void Process()
+    {
+      sem.Post();
+    };
+
+    //------------------------------------------------------------------------
+    // Handle an event other that a message arrival
+    //------------------------------------------------------------------------
+    virtual uint8_t OnStreamEvent( StreamEvent          event,
+                                   XrdCl::XRootDStatus  status )
+    {
+      if( event == Ready )
+        return 0;
+      this->status = status;
+      sem.Post();
+      return RemoveHandler;
+    };
+
+    //------------------------------------------------------------------------
+    // The requested action has been performed and the status is available
+    //------------------------------------------------------------------------
+    virtual void OnStatusReady( const XrdCl::Message *message,
+                                XrdCl::XRootDStatus   status )
+    {
+      request = message;
+      this->status = status;
+      if( !status.IsOK() )
+        sem.Post();
+    }
+
+    //------------------------------------------------------------------------
+    // Get a timestamp after which we give up
+    //------------------------------------------------------------------------
+    virtual time_t GetExpiration()
+    {
+      return expiration;
+    }
+
+    void SetExpiration( time_t e )
+    {
+      expiration = e;
+    }
+
+    XrdCl::XRootDStatus WaitFor( XrdCl::Message &rsp )
+    {
+      sem.Wait();
+      if( response )
+        rsp = std::move( *response );
+      return status;
+    }
+
+    void SetFilter( unsigned char id0 = 0, unsigned char id1 = 0 )
+    {
+      filter.streamId[0] = id0;
+      filter.streamId[1] = id1;
+    }
+};
+
+//------------------------------------------------------------------------------
+// Thread argument passing helper
+//------------------------------------------------------------------------------
+struct ArgHelper
+{
+  XrdCl::PostMaster *pm;
+  int                index;
+};
+
+//------------------------------------------------------------------------------
+// Post master test thread
+//------------------------------------------------------------------------------
+void *TestThreadFunc( void *arg )
+{
+  using namespace XrdCl;
+
+  std::string address;
+  Env *testEnv = TestEnv::GetEnv();
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+
+  ArgHelper *a = (ArgHelper*)arg;
+  URL        host( address );
+
+  //----------------------------------------------------------------------------
+  // Send the ping messages
+  //----------------------------------------------------------------------------
+  SyncMsgHandler msgHandlers[100];
+  Message        msgs[100];
+  time_t expires = time(0)+1200;
+  for( int i = 0; i < 100; ++i )
+  {
+    msgs[i].Allocate( sizeof( ClientPingRequest ) );
+    msgs[i].Zero();
+    msgs[i].SetDescription( "kXR_ping ()" );
+    ClientPingRequest *request = (ClientPingRequest *)msgs[i].GetBuffer();
+    request->streamid[0] = a->index;
+    request->requestid   = kXR_ping;
+    request->dlen        = 0;
+    XRootDTransport::MarshallRequest( &msgs[i] );
+    request->streamid[1] = i;
+    msgHandlers[i].SetFilter( a->index, i );
+    msgHandlers[i].SetExpiration( expires );
+    GTEST_ASSERT_XRDST( a->pm->Send( host, &msgs[i], &msgHandlers[i], false, expires ) );
+  }
+
+  //----------------------------------------------------------------------------
+  // Receive the answers
+  //----------------------------------------------------------------------------
+  for( int i = 0; i < 100; ++i )
+  {
+    XrdCl::Message msg;
+    GTEST_ASSERT_XRDST( msgHandlers[i].WaitFor( msg ) );
+    ServerResponse *resp = (ServerResponse *)msg.GetBuffer();
+    EXPECT_TRUE( resp );
+    EXPECT_EQ( resp->hdr.status, kXR_ok );
+    EXPECT_EQ( msg.GetSize(), 8 );
+  }
+  return 0;
+}
+
+//------------------------------------------------------------------------------
+// Threading test
+//------------------------------------------------------------------------------
+TEST(PostMasterTest, ThreadingTest)
+{
+  using namespace XrdCl;
+  PostMasterFetch pmfetch;
+  PostMaster *postMaster = pmfetch.Get();
+
+  pthread_t thread[100];
+  ArgHelper helper[100];
+
+  for( int i = 0; i < 100; ++i )
+  {
+    helper[i].pm    = postMaster;
+    helper[i].index = i;
+    pthread_create( &thread[i], 0, TestThreadFunc, &helper[i] );
+  }
+
+  for( int i = 0; i < 100; ++i )
+    pthread_join( thread[i], 0 );
+}
+
+//------------------------------------------------------------------------------
+// Test the functionality of a poller
+//------------------------------------------------------------------------------
+TEST(PostMasterTest, FunctionalTest)
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize the stuff
+  //----------------------------------------------------------------------------
+  Env *env = DefaultEnv::GetEnv();
+  Env *testEnv = TestEnv::GetEnv();
+  env->PutInt( "TimeoutResolution", 1 );
+  env->PutInt( "ConnectionWindow",  5 );
+
+  PostMasterFetch pmfetch;
+  PostMaster *postMaster = pmfetch.Get();
+
+  std::string address;
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+
+  //----------------------------------------------------------------------------
+  // Send a message and wait for the answer
+  //----------------------------------------------------------------------------
+  time_t    expires = ::time(0)+60;
+  Message   m1, m2;
+  URL       host( address );
+
+  SyncMsgHandler msgHandler1;
+  msgHandler1.SetFilter( 1, 2 );
+  msgHandler1.SetExpiration( expires );
+
+  m1.Allocate( sizeof( ClientPingRequest ) );
+  m1.Zero();
+
+  ClientPingRequest *request = (ClientPingRequest *)m1.GetBuffer();
+  request->streamid[0] = 1;
+  request->streamid[1] = 2;
+  request->requestid   = kXR_ping;
+  request->dlen        = 0;
+  XRootDTransport::MarshallRequest( &m1 );
+
+  GTEST_ASSERT_XRDST( postMaster->Send( host, &m1, &msgHandler1, false, expires ) );
+
+  GTEST_ASSERT_XRDST( msgHandler1.WaitFor( m2 ) );
+  ServerResponse *resp = (ServerResponse *)m2.GetBuffer();
+  EXPECT_TRUE( resp );
+  EXPECT_EQ( resp->hdr.status, kXR_ok );
+  EXPECT_EQ( m2.GetSize(), 8 );
+
+  //----------------------------------------------------------------------------
+  // Send out some stuff to a location where nothing listens
+  //----------------------------------------------------------------------------
+  env->PutInt( "ConnectionWindow", 5 );
+  env->PutInt( "ConnectionRetry", 3 );
+  URL localhost1( "root://localhost:10101" );
+
+  SyncMsgHandler msgHandler2;
+  msgHandler2.SetFilter( 1, 2 );
+  time_t shortexp = ::time(0) + 1;
+  msgHandler2.SetExpiration( shortexp );
+  GTEST_ASSERT_XRDST( postMaster->Send( localhost1, &m1, &msgHandler2, false,
+                        shortexp ) );
+  GTEST_ASSERT_XRDST_NOTOK( msgHandler2.WaitFor( m2 ), errOperationExpired );
+
+  SyncMsgHandler msgHandler3;
+  msgHandler3.SetFilter( 1, 2 );
+  msgHandler3.SetExpiration( expires );
+  GTEST_ASSERT_XRDST( postMaster->Send( localhost1, &m1, &msgHandler3, false,
+                                          expires ) );
+  GTEST_ASSERT_XRDST_NOTOK( msgHandler3.WaitFor( m2 ), errConnectionError );
+
+  //----------------------------------------------------------------------------
+  // Test the transport queries
+  //----------------------------------------------------------------------------
+  AnyObject nameObj, sidMgrObj;
+  Status st1, st2;
+  const char *name   = 0;
+
+  GTEST_ASSERT_XRDST( postMaster->QueryTransport( host,
+                                                   TransportQuery::Name,
+                                                   nameObj ) );
+  nameObj.Get( name );
+
+  EXPECT_TRUE( name );
+  EXPECT_TRUE( !::strcmp( name, "XRootD" ) );
+
+  //----------------------------------------------------------------------------
+  // Reinitialize and try to do something
+  //----------------------------------------------------------------------------
+  env->PutInt( "LoadBalancerTTL", 5 );
+  postMaster = pmfetch.Reset();
+
+  m2.Free();
+  m1.Zero();
+
+  request = (ClientPingRequest *)m1.GetBuffer();
+  request->streamid[0] = 1;
+  request->streamid[1] = 2;
+  request->requestid   = kXR_ping;
+  request->dlen        = 0;
+  XRootDTransport::MarshallRequest( &m1 );
+
+  SyncMsgHandler msgHandler4;
+  msgHandler4.SetFilter( 1, 2 );
+  msgHandler4.SetExpiration( expires );
+  GTEST_ASSERT_XRDST( postMaster->Send( host, &m1, &msgHandler4, false, expires ) );
+
+  GTEST_ASSERT_XRDST( msgHandler4.WaitFor( m2 ) );
+  resp = (ServerResponse *)m2.GetBuffer();
+  EXPECT_TRUE( resp );
+  EXPECT_EQ( resp->hdr.status, kXR_ok );
+  EXPECT_EQ( m2.GetSize(), 8 );
+
+  //----------------------------------------------------------------------------
+  // Sleep 10 secs waiting for iddle connection to be closed and see
+  // whether we can reconnect
+  //----------------------------------------------------------------------------
+  sleep( 10 );
+  SyncMsgHandler msgHandler5;
+  msgHandler5.SetFilter( 1, 2 );
+  msgHandler5.SetExpiration( expires );
+  GTEST_ASSERT_XRDST( postMaster->Send( host, &m1, &msgHandler5, false, expires ) );
+
+  GTEST_ASSERT_XRDST( msgHandler5.WaitFor( m2 ) );
+  resp = (ServerResponse *)m2.GetBuffer();
+  EXPECT_TRUE( resp );
+  EXPECT_EQ( resp->hdr.status, kXR_ok );
+  EXPECT_EQ( m2.GetSize(), 8 );
+}
+
+
+//------------------------------------------------------------------------------
+// Test the functionality of a poller
+//------------------------------------------------------------------------------
+TEST(PostMasterTest, PingIPv6)
+{
+  using namespace XrdCl;
+#if 0
+  //----------------------------------------------------------------------------
+  // Initialize the stuff
+  //----------------------------------------------------------------------------
+  PostMasterFetch pmfetch;
+  PostMaster *postMaster = pmfetch.Get();
+
+  //----------------------------------------------------------------------------
+  // Build the message
+  //----------------------------------------------------------------------------
+  Message   m1, *m2 = 0;
+  XrdFilter f1( 1, 2 );
+  URL       localhost1( "root://[::1]" );
+  URL       localhost2( "root://[::127.0.0.1]" );
+
+  m1.Allocate( sizeof( ClientPingRequest ) );
+  m1.Zero();
+
+  ClientPingRequest *request = (ClientPingRequest *)m1.GetBuffer();
+  request->streamid[0] = 1;
+  request->streamid[1] = 2;
+  request->requestid   = kXR_ping;
+  request->dlen        = 0;
+  XRootDTransport::MarshallRequest( &m1 );
+
+  Status sc;
+
+  //----------------------------------------------------------------------------
+  // Send the message - localhost1
+  //----------------------------------------------------------------------------
+  sc = postMaster->Send( localhost1, &m1, false, 1200 );
+  EXPECT_TRUE( sc.IsOK() );
+
+  sc = postMaster->Receive( localhost1, m2, &f1, false, 1200 );
+  EXPECT_TRUE( sc.IsOK() );
+  ServerResponse *resp = (ServerResponse *)m2->GetBuffer();
+  EXPECT_TRUE( resp );
+  EXPECT_EQ( resp->hdr.status, kXR_ok );
+  EXPECT_EQ( m2->GetSize(), 8 );
+
+  //----------------------------------------------------------------------------
+  // Send the message - localhost2
+  //----------------------------------------------------------------------------
+  sc = postMaster->Send( localhost2, &m1, false, 1200 );
+  EXPECT_TRUE( sc.IsOK() );
+
+  sc = postMaster->Receive( localhost2, m2, &f1, 1200 );
+  EXPECT_TRUE( sc.IsOK() );
+  resp = (ServerResponse *)m2->GetBuffer();
+  EXPECT_TRUE( resp );
+  EXPECT_EQ( resp->hdr.status, kXR_ok );
+  EXPECT_EQ( m2->GetSize(), 8 );
+#endif
+}
+
+namespace
+{
+  //----------------------------------------------------------------------------
+  // Create a ping message
+  //----------------------------------------------------------------------------
+  // XrdCl::Message *CreatePing( char streamID1, char streamID2 )
+  // {
+  //   using namespace XrdCl;
+  //   Message *m = new Message();
+  //   m->Allocate( sizeof( ClientPingRequest ) );
+  //   m->Zero();
+
+  //   ClientPingRequest *request = (ClientPingRequest *)m->GetBuffer();
+  //   request->streamid[0] = streamID1;
+  //   request->streamid[1] = streamID2;
+  //   request->requestid   = kXR_ping;
+  //   XRootDTransport::MarshallRequest( m );
+  //   return m;
+  // }
+}
+
+
+//------------------------------------------------------------------------------
+// Connection test
+//------------------------------------------------------------------------------
+
+#if 0
+TEST(PostMasterTest, MultiIPConnectionTest)
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize the stuff
+  //----------------------------------------------------------------------------
+  Env *env = DefaultEnv::GetEnv();
+  Env *testEnv = TestEnv::GetEnv();
+  env->PutInt( "TimeoutResolution", 1 );
+  env->PutInt( "ConnectionWindow",  5 );
+
+  PostMasterFetch pmfetch;
+  PostMaster *postMaster = pmfetch.Get();
+
+  std::string address;
+  EXPECT_TRUE( testEnv->GetString( "MultiIPServerURL", address ) );
+
+  time_t expires = ::time(0)+1200;
+  URL url1( "nenexistent" );
+  URL url2( address );
+  URL url3( address );
+  url2.SetPort( 1111 );
+  url3.SetPort( 1099 );
+
+  //----------------------------------------------------------------------------
+  // Sent ping to a nonexistent host
+  //----------------------------------------------------------------------------
+  SyncMsgHandler msgHandler1;
+  msgHandler1.SetFilter( 1, 2 );
+  msgHandler1.SetExpiration( expires );
+  Message *m = CreatePing( 1, 2 );
+  GTEST_ASSERT_XRDST_NOTOK( postMaster->Send( url1, m, &msgHandler1, false, expires ),
+                              errInvalidAddr );
+
+  //----------------------------------------------------------------------------
+  // Try on the wrong port
+  //----------------------------------------------------------------------------
+  SyncMsgHandler msgHandler2;
+  msgHandler2.SetFilter( 1, 2 );
+  msgHandler2.SetExpiration( expires );
+  Message m2;
+
+  GTEST_ASSERT_XRDST( postMaster->Send( url2, m, &msgHandler2, false, expires ) );
+  GTEST_ASSERT_XRDST_NOTOK( msgHandler2.WaitFor( m2 ), errConnectionError );
+
+  //----------------------------------------------------------------------------
+  // Try on a good one
+  //----------------------------------------------------------------------------
+  SyncMsgHandler msgHandler3;
+  msgHandler3.SetFilter( 1, 2 );
+  msgHandler3.SetExpiration( expires );
+
+  GTEST_ASSERT_XRDST( postMaster->Send( url3, m, &msgHandler3, false, expires ) );
+  GTEST_ASSERT_XRDST( msgHandler3.WaitFor( m2 ) );
+  ServerResponse *resp = (ServerResponse *)m2.GetBuffer();
+  EXPECT_TRUE( resp );
+  EXPECT_EQ( resp->hdr.status, kXR_ok );
+  EXPECT_EQ( m2.GetSize(), 8 );
+}
+#endif
diff --git a/tests/XrdCl/XrdClSocket.cc b/tests/XrdCl/XrdClSocket.cc
new file mode 100644
index 00000000000..df8f433bff5
--- /dev/null
+++ b/tests/XrdCl/XrdClSocket.cc
@@ -0,0 +1,304 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "Server.hh"
+#include "Utils.hh"
+#include "TestEnv.hh"
+#include "XrdCl/XrdClSocket.hh"
+#include "XrdCl/XrdClUtils.hh"
+
+using namespace XrdClTests;
+
+//------------------------------------------------------------------------------
+// Mock socket for testing
+//------------------------------------------------------------------------------
+struct MockSocket : public XrdCl::Socket
+{
+  public:
+
+    MockSocket() : size( sizeof( ServerResponseHeader ) + sizeof( ServerResponseBody_Protocol ) ),
+                   buffer( reinterpret_cast( &response ) ), offset( 0 ),
+                   random_engine( std::chrono::system_clock::now().time_since_epoch().count() ),
+                   retrygen( 0, 9 ),
+                   retry_threshold( retrygen( random_engine ) )
+    {
+      response.hdr.status = kXR_ok;
+      response.hdr.streamid[0] = 1;
+      response.hdr.streamid[1] = 2;
+      response.hdr.dlen = htonl( sizeof( ServerResponseBody_Protocol ) );
+
+      response.body.protocol.flags = 123;
+      response.body.protocol.pval  = 4567;
+      response.body.protocol.secreq.rsvd   = 'A';
+      response.body.protocol.secreq.seclvl = 'B';
+      response.body.protocol.secreq.secopt = 'C';
+      response.body.protocol.secreq.secver = 'D';
+      response.body.protocol.secreq.secvsz = 'E';
+      response.body.protocol.secreq.theTag = 'F';
+      response.body.protocol.secreq.secvec.reqindx = 'G';
+      response.body.protocol.secreq.secvec.reqsreq = 'H';
+    }
+
+    virtual XrdCl::XRootDStatus Read( char *outbuf, size_t rdsize, int &bytesRead )
+    {
+      size_t btsleft = size - offset;
+      if( btsleft == 0 || nodata() )
+        return XrdCl::XRootDStatus( XrdCl::stOK, XrdCl::suRetry );
+
+      if( rdsize > btsleft )
+        rdsize = btsleft;
+
+      std::uniform_int_distribution sizegen( 0, rdsize );
+      rdsize = sizegen( random_engine );
+
+      if( rdsize == 0 )
+        return XrdCl::XRootDStatus( XrdCl::stOK, XrdCl::suRetry );
+
+      memcpy( outbuf, buffer + offset, rdsize );
+      offset += rdsize;
+      bytesRead = rdsize;
+
+      return XrdCl::XRootDStatus();
+    }
+
+    virtual XrdCl::XRootDStatus Send( const char *buffer, size_t size, int &bytesWritten )
+    {
+      return XrdCl::XRootDStatus( XrdCl::stError, XrdCl::errNotSupported );
+    }
+
+    inline bool IsEqual( XrdCl::Message &msg )
+    {
+      response.hdr.dlen = ntohl( response.hdr.dlen );
+      bool ok = ( memcmp( msg.GetBuffer(), &response, size ) == 0 );
+      response.hdr.dlen = htonl( response.hdr.dlen );
+      return ok;
+    }
+
+  private:
+
+    inline bool nodata()
+    {
+      size_t doretry = retrygen( random_engine );
+      return doretry > retry_threshold;
+    }
+
+    ServerResponse response;
+    const size_t size;
+    char *buffer;
+    size_t offset;
+
+    std::default_random_engine random_engine;
+    std::uniform_int_distribution retrygen;
+    const size_t retry_threshold;
+};
+
+//------------------------------------------------------------------------------
+// Client handler
+//------------------------------------------------------------------------------
+class RandomHandler: public ClientHandler
+{
+  public:
+    virtual void HandleConnection( int socket )
+    {
+      XrdCl::ScopedDescriptor scopedDesc( socket );
+      XrdCl::Log *log = TestEnv::GetLog();
+
+      //------------------------------------------------------------------------
+      // Pump some data
+      //------------------------------------------------------------------------
+      uint8_t  packets = random() % 100;
+      uint16_t packetSize;
+      char     buffer[50000];
+      log->Debug( 1, "Sending %d packets to the client", packets );
+
+      if( ::Utils::Write( socket, &packets, 1 ) != 1 )
+      {
+        log->Error( 1, "Unable to send the packet count" );
+        return;
+      }
+
+      for( int i = 0; i < packets; ++i )
+      {
+        packetSize = random() % 50000;
+        log->Dump( 1, "Sending %d packet, %d bytes of data", i, packetSize );
+        if( Utils::GetRandomBytes( buffer, packetSize ) != packetSize )
+        {
+          log->Error( 1, "Unable to get %d bytes of random data", packetSize );
+          return;
+        }
+
+        if( ::Utils::Write( socket, &packetSize, 2 ) != 2 )
+        {
+          log->Error( 1, "Unable to send the packet size" );
+          return;
+        }
+        if( ::Utils::Write( socket, buffer, packetSize ) != packetSize )
+        {
+          log->Error( 1, "Unable to send the %d bytes of random data",
+                      packetSize );
+          return;
+        }
+        UpdateSentData( buffer, packetSize );
+      }
+
+      //------------------------------------------------------------------------
+      // Receive some data
+      //------------------------------------------------------------------------
+      if( ::Utils::Read( socket, &packets, 1 ) != 1 )
+      {
+        log->Error( 1, "Unable to receive the packet count" );
+        return;
+      }
+
+      log->Debug( 1, "Receivng %d packets from the client", packets );
+
+      for( int i = 0; i < packets; ++i )
+      {
+        if( ::Utils::Read( socket, &packetSize, 2 ) != 2 )
+        {
+          log->Error( 1, "Unable to receive the packet size" );
+          return;
+        }
+
+        if ( ::Utils::Read( socket, buffer, packetSize ) != packetSize )
+        {
+          log->Error( 1, "Unable to receive the %d bytes of data",
+                      packetSize );
+          return;
+        }
+        UpdateReceivedData( buffer, packetSize );
+        log->Dump( 1, "Received %d bytes from the client", packetSize );
+      }
+    }
+};
+
+//------------------------------------------------------------------------------
+// Client handler factory
+//------------------------------------------------------------------------------
+class RandomHandlerFactory: public ClientHandlerFactory
+{
+  public:
+    virtual ClientHandler *CreateHandler()
+    {
+      return new RandomHandler();
+    }
+};
+
+//------------------------------------------------------------------------------
+// SocketTest class declaration
+//------------------------------------------------------------------------------
+class SocketTest : public ::testing::Test {};
+
+//------------------------------------------------------------------------------
+// Test the transfer
+//------------------------------------------------------------------------------
+TEST(SocketTest, TransferTest)
+{
+  using namespace XrdCl;
+  srandom( time(0) );
+  Server serv( Server::Both );
+  Socket sock;
+
+  //----------------------------------------------------------------------------
+  // Start up the server and connect to it
+  //----------------------------------------------------------------------------
+  uint16_t port = 9998; // was 9999, but we need to change ports from other
+                    // tests so that we can run all of them in parallel. 
+                    // Will find another, better way to ensure this in the future 
+  EXPECT_TRUE( serv.Setup( port, 1, new RandomHandlerFactory() ) );
+  EXPECT_TRUE( serv.Start() );
+
+  EXPECT_EQ( sock.GetStatus(), Socket::Disconnected );
+  EXPECT_TRUE( sock.Initialize( AF_INET6 ).IsOK() );
+  EXPECT_TRUE( sock.Connect( "localhost", port ).IsOK() );
+  EXPECT_EQ( sock.GetStatus(), Socket::Connected );
+
+  //----------------------------------------------------------------------------
+  // Get the number of packets
+  //----------------------------------------------------------------------------
+  uint8_t  packets;
+  uint32_t bytesTransmitted;
+  uint16_t packetSize;
+  Status   sc;
+  char     buffer[50000];
+  uint64_t sentCounter = 0;
+  uint32_t sentChecksum = ::Utils::ComputeCRC32( 0, 0 );
+  uint64_t receivedCounter = 0;
+  uint32_t receivedChecksum = ::Utils::ComputeCRC32( 0, 0 );
+  sc = sock.ReadRaw( &packets, 1, 60, bytesTransmitted );
+  EXPECT_EQ( sc.status, stOK );
+
+  //----------------------------------------------------------------------------
+  // Read each packet
+  //----------------------------------------------------------------------------
+  for( int i = 0; i < packets; ++i )
+  {
+    sc = sock.ReadRaw( &packetSize, 2, 60, bytesTransmitted );
+    EXPECT_EQ( sc.status, stOK );
+    sc = sock.ReadRaw( buffer, packetSize, 60, bytesTransmitted );
+    EXPECT_EQ( sc.status, stOK );
+    receivedCounter += bytesTransmitted;
+    receivedChecksum = ::Utils::UpdateCRC32( receivedChecksum, buffer,
+                                             bytesTransmitted );
+  }
+
+  //----------------------------------------------------------------------------
+  // Send the number of packets
+  //----------------------------------------------------------------------------
+  packets = random() % 100;
+
+  sc = sock.WriteRaw( &packets, 1, 60, bytesTransmitted );
+  EXPECT_EQ( sc.status, stOK );
+  EXPECT_EQ( bytesTransmitted, 1 );
+
+  for( int i = 0; i < packets; ++i )
+  {
+    packetSize = random() % 50000;
+    EXPECT_EQ( ::Utils::GetRandomBytes( buffer, packetSize ), packetSize );
+
+    sc = sock.WriteRaw( (char *)&packetSize, 2, 60, bytesTransmitted );
+    EXPECT_EQ( sc.status, stOK );
+    EXPECT_EQ( bytesTransmitted, 2 );
+    sc = sock.WriteRaw( buffer, packetSize, 60, bytesTransmitted );
+    EXPECT_EQ( sc.status, stOK );
+    EXPECT_EQ( bytesTransmitted, packetSize );
+    sentCounter += bytesTransmitted;
+    sentChecksum = ::Utils::UpdateCRC32( sentChecksum, buffer,
+                                         bytesTransmitted );
+  }
+
+  //----------------------------------------------------------------------------
+  // Check the counters and the checksums
+  //----------------------------------------------------------------------------
+  std::string socketName = sock.GetSockName();
+
+  sock.Close();
+  EXPECT_TRUE( serv.Stop() );
+
+  std::pair sent     = serv.GetSentStats( socketName );
+  std::pair received = serv.GetReceivedStats( socketName );
+  EXPECT_EQ( sentCounter, received.first );
+  EXPECT_EQ( receivedCounter, sent.first );
+  EXPECT_EQ( sentChecksum, received.second );
+  EXPECT_EQ( receivedChecksum, sent.second );
+}
diff --git a/tests/XrdCl/XrdClThreadingTest.cc b/tests/XrdCl/XrdClThreadingTest.cc
new file mode 100644
index 00000000000..baee052331d
--- /dev/null
+++ b/tests/XrdCl/XrdClThreadingTest.cc
@@ -0,0 +1,336 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+#include "TestEnv.hh"
+#include "Utils.hh"
+#include "GTestXrdHelpers.hh"
+#include "XrdCl/XrdClFile.hh"
+#include "XrdCl/XrdClDefaultEnv.hh"
+#include "XrdCl/XrdClUtils.hh"
+#include 
+#include 
+#include 
+#include 
+#include 
+#include "XrdCks/XrdCksData.hh"
+
+//------------------------------------------------------------------------------
+// Thread helper struct
+//------------------------------------------------------------------------------
+struct ThreadData
+{
+  ThreadData():
+    file( 0 ), startOffset( 0 ), length( 0 ), checkSum( 0 ),
+    firstBlockChecksum(0) {}
+  XrdCl::File *file;
+  uint64_t     startOffset;
+  uint64_t     length;
+  uint32_t     checkSum;
+  uint32_t     firstBlockChecksum;
+};
+
+const uint32_t MB = 1024*1024;
+
+//------------------------------------------------------------------------------
+// Declaration
+//------------------------------------------------------------------------------
+class ThreadingTest: public ::testing::Test
+{
+  public:
+    typedef void (*TransferCallback)( ThreadData *data );
+    void ReadTestFunc( TransferCallback transferCallback );
+    void ReadTest();
+    void MultiStreamReadTest();
+    void ReadForkTest();
+    void MultiStreamReadForkTest();
+    void MultiStreamReadMonitorTest();
+};
+
+//------------------------------------------------------------------------------
+// Reader thread
+//------------------------------------------------------------------------------
+void *DataReader( void *arg )
+{
+  using namespace XrdClTests;
+
+  ThreadData *td = (ThreadData*)arg;
+
+  uint64_t  offset    = td->startOffset;
+  uint64_t  dataLeft  = td->length;
+  uint64_t  chunkSize = 0;
+  uint32_t  bytesRead = 0;
+  char     *buffer    = new char[4*MB];
+
+  while( 1 )
+  {
+    chunkSize = 4*MB;
+    if( chunkSize > dataLeft )
+      chunkSize = dataLeft;
+
+    if( chunkSize == 0 )
+      break;
+
+    GTEST_ASSERT_XRDST( td->file->Read( offset, chunkSize, buffer,
+                                          bytesRead ) );
+
+    offset   += bytesRead;
+    dataLeft -= bytesRead;
+    td->checkSum = Utils::UpdateCRC32( td->checkSum, buffer, bytesRead );
+  }
+
+  delete [] buffer;
+
+  return 0;
+}
+
+//------------------------------------------------------------------------------
+// Read test
+//------------------------------------------------------------------------------
+void ThreadingTest::ReadTestFunc( TransferCallback transferCallback )
+{
+  using namespace XrdCl;
+
+  //----------------------------------------------------------------------------
+  // Initialize
+  //----------------------------------------------------------------------------
+  Env *testEnv = XrdClTests::TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  URL url( address );
+  EXPECT_TRUE( url.IsValid() );
+
+  std::string fileUrl[5];
+  std::string path[5];
+  path[0] = dataPath + "/1db882c8-8cd6-4df1-941f-ce669bad3458.dat";
+  path[1] = dataPath + "/3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat";
+  path[2] = dataPath + "/7235b5d1-cede-4700-a8f9-596506b4cc38.dat";
+  path[3] = dataPath + "/7e480547-fe1a-4eaf-a210-0f3927751a43.dat";
+  path[4] = dataPath + "/89120cec-5244-444c-9313-703e4bee72de.dat";
+
+  for( int i = 0; i < 5; ++i )
+    fileUrl[i] = address + "/" + path[i];
+
+  //----------------------------------------------------------------------------
+  // Open and stat the files
+  //----------------------------------------------------------------------------
+  ThreadData threadData[20];
+
+  for( int i = 0; i < 5; ++i )
+  {
+    File     *f = new File();
+    StatInfo *si = 0;
+    GTEST_ASSERT_XRDST( f->Open( fileUrl[i], OpenFlags::Read ) );
+    GTEST_ASSERT_XRDST( f->Stat( false, si ) );
+    EXPECT_TRUE( si );
+    EXPECT_TRUE( si->TestFlags( StatInfo::IsReadable ) );
+
+    uint64_t step = si->GetSize()/4;
+
+    for( int j = 0; j < 4; ++j )
+    {
+      threadData[j*5+i].file        = f;
+      threadData[j*5+i].startOffset = j*step;
+      threadData[j*5+i].length      = step;
+      threadData[j*5+i].checkSum    = XrdClTests::Utils::GetInitialCRC32();
+
+
+      //------------------------------------------------------------------------
+      // Get the checksum of the first 4MB block at the startOffser - this
+      // will be verified by the forking test
+      //------------------------------------------------------------------------
+      uint64_t  offset = threadData[j*5+i].startOffset;
+      char     *buffer    = new char[4*MB];
+      uint32_t  bytesRead = 0;
+
+      GTEST_ASSERT_XRDST( f->Read( offset, 4*MB, buffer, bytesRead ) );
+      EXPECT_EQ( bytesRead, 4*MB );
+      threadData[j*5+i].firstBlockChecksum =
+        XrdClTests::Utils::ComputeCRC32( buffer, 4*MB );
+      delete [] buffer;
+    }
+
+    threadData[15+i].length = si->GetSize() - threadData[15+i].startOffset;
+    delete si;
+  }
+
+  //----------------------------------------------------------------------------
+  // Spawn the threads and wait for them to finish
+  //----------------------------------------------------------------------------
+  pthread_t thread[20];
+  for( int i = 0; i < 20; ++i )
+    GTEST_ASSERT_PTHREAD( pthread_create( &(thread[i]), 0,
+                            ::DataReader, &(threadData[i]) ) );
+
+  if( transferCallback )
+    (*transferCallback)( threadData );
+
+  for( int i = 0; i < 20; ++i )
+    GTEST_ASSERT_PTHREAD( pthread_join( thread[i], 0 ) );
+
+  //----------------------------------------------------------------------------
+  // Glue up and compare the checksums
+  //----------------------------------------------------------------------------
+  uint32_t checkSums[5];
+  for( int i = 0; i < 5; ++i )
+  {
+    //--------------------------------------------------------------------------
+    // Calculate the local check sum
+    //--------------------------------------------------------------------------
+    checkSums[i] = threadData[i].checkSum;
+    for( int j = 1; j < 4; ++j )
+    {
+      checkSums[i] = XrdClTests::Utils::CombineCRC32( checkSums[i],
+                                          threadData[j*5+i].checkSum,
+                                          threadData[j*5+i].length );
+    }
+
+    char crcBuff[9];
+    XrdCksData crc; crc.Set( &checkSums[i], 4 ); crc.Get( crcBuff, 9 );
+    std::string transferSum = "zcrc32:"; transferSum += crcBuff;
+
+    //--------------------------------------------------------------------------
+    // Get the checksum
+    //--------------------------------------------------------------------------
+    std::string remoteSum, lastUrl;
+    threadData[i].file->GetProperty( "LastURL", lastUrl );
+    GTEST_ASSERT_XRDST( Utils::GetRemoteCheckSum(
+                            remoteSum, "zcrc32", lastUrl ) );
+    EXPECT_EQ( remoteSum, transferSum ) << path[i];
+  }
+
+  //----------------------------------------------------------------------------
+  // Close the files
+  //----------------------------------------------------------------------------
+  for( int i = 0; i < 5; ++i )
+  {
+    GTEST_ASSERT_XRDST( threadData[i].file->Close() );
+    delete threadData[i].file;
+  }
+}
+
+
+//------------------------------------------------------------------------------
+// Read test
+//------------------------------------------------------------------------------
+TEST_F(ThreadingTest, ReadTest)
+{
+  ReadTestFunc(0);
+}
+
+//------------------------------------------------------------------------------
+// Multistream read test
+//------------------------------------------------------------------------------
+TEST_F(ThreadingTest, MultiStreamReadTest)
+{
+  XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv();
+  env->PutInt( "SubStreamsPerChannel", 4 );
+  ReadTestFunc(0);
+}
+
+//------------------------------------------------------------------------------
+// Child - read some data from each of the open files and close them
+//------------------------------------------------------------------------------
+int runChild( ThreadData *td )
+{
+  XrdCl::Log *log = XrdClTests::TestEnv::GetLog();
+  log->Debug( 1, "Running the child" );
+
+  for( int i = 0; i < 20; ++i )
+  {
+    uint64_t  offset    = td[i].startOffset;
+    char     *buffer    = new char[4*MB];
+    uint32_t  bytesRead = 0;
+
+    GTEST_ASSERT_XRDST( td[i].file->Read( offset, 4*MB, buffer, bytesRead ) );
+    EXPECT_EQ( bytesRead, 4*MB );
+    EXPECT_EQ( td[i].firstBlockChecksum,
+                    XrdClTests::Utils::ComputeCRC32( buffer, 4*MB ) );
+    delete [] buffer;
+  }
+
+  for( int i = 0; i < 5; ++i )
+  {
+    GTEST_ASSERT_XRDST( td[i].file->Close() );
+    delete td[i].file;
+  }
+
+  return 0;
+}
+
+//------------------------------------------------------------------------------
+// Forking function
+//------------------------------------------------------------------------------
+void forkAndRead( ThreadData *data )
+{
+  XrdCl::Log *log = XrdClTests::TestEnv::GetLog();
+  for( int chld = 0; chld < 5; ++chld )
+  {
+    sleep(1);
+    pid_t pid;
+    log->Debug( 1, "About to fork" );
+    GTEST_ASSERT_ERRNO( (pid=fork()) != -1 );
+
+    if( !pid )  _exit( runChild( data ) );
+
+    log->Debug( 1, "Forked successfully, pid of the child: %d", pid );
+    int status;
+    log->Debug( 1, "Waiting for the child" );
+    GTEST_ASSERT_ERRNO( waitpid( pid, &status, 0 ) != -1 );
+    log->Debug( 1, "Wait done, status: %d", status );
+    EXPECT_TRUE( WIFEXITED( status ) );
+    EXPECT_EQ( WEXITSTATUS( status ), 0 );
+  }
+}
+
+//------------------------------------------------------------------------------
+// Read fork test
+//------------------------------------------------------------------------------
+TEST_F(ThreadingTest, ReadForkTest)
+{
+  XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv();
+  env->PutInt( "RunForkHandler", 1 );
+  ReadTestFunc(&forkAndRead);
+}
+
+//------------------------------------------------------------------------------
+// Multistream read fork test
+//------------------------------------------------------------------------------
+TEST_F(ThreadingTest, MultiStreamReadForkTest)
+{
+  XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv();
+  env->PutInt( "SubStreamsPerChannel", 4 );
+  env->PutInt( "RunForkHandler", 1 );
+  ReadTestFunc(&forkAndRead);
+}
+
+//------------------------------------------------------------------------------
+// Multistream read monitor
+//------------------------------------------------------------------------------
+TEST_F(ThreadingTest, MultiStreamReadMonitorTest)
+{
+  XrdCl::Env *env = XrdCl::DefaultEnv::GetEnv();
+  env->PutString( "ClientMonitor", "./libXrdClTestMonitor.so" );
+  env->PutString( "ClientMonitorParam", "TestParam" );
+  env->PutInt( "SubStreamsPerChannel", 4 );
+  ReadTestFunc(0);
+}
diff --git a/tests/XrdCl/XrdClURL.cc b/tests/XrdCl/XrdClURL.cc
index 24e7fdc2aad..046f41a8c8f 100644
--- a/tests/XrdCl/XrdClURL.cc
+++ b/tests/XrdCl/XrdClURL.cc
@@ -1,6 +1,8 @@
 #undef NDEBUG
 
 #include 
+#include "XrdSys/XrdSysPlatform.hh"
+
 #include 
 
 #include 
@@ -14,7 +16,7 @@ class URLTest : public ::testing::Test {};
 
 TEST(URLTest, LocalURLs)
 {
-  char url[PATH_MAX];
+  char url[MAXPATHLEN];
   for (const auto& protocol : { "", "file://" }) {
     for (const auto& path : { "/dev", "/dev/", "/dev/null" }) {
       for (const auto& params : { "", "?param=value", "?param1=value1¶m2=value2" }) {
@@ -44,8 +46,8 @@ TEST(URLTest, LocalURLs)
 
 TEST(URLTest, RemoteURLs)
 {
-  char url[PATH_MAX];
-  char path_params[PATH_MAX];
+  char url[MAXPATHLEN];
+  char path_params[MAXPATHLEN];
   for (const char *protocol : { "", "http", "root", "https", "roots" }) {
     int default_port = *protocol == 'h' ? (strlen(protocol) == 4 ? 80 : 443) : 1094;
     for (const char *user : { "", "alice", "bob", "user_123", "xrootd" }) {
diff --git a/tests/XrdCl/XrdClUtilsTest.cc b/tests/XrdCl/XrdClUtilsTest.cc
new file mode 100644
index 00000000000..7ba30ec8676
--- /dev/null
+++ b/tests/XrdCl/XrdClUtilsTest.cc
@@ -0,0 +1,252 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// This file is part of the XRootD software suite.
+//
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//
+// In applying this licence, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+//------------------------------------------------------------------------------
+
+#include 
+#include "XrdCl/XrdClAnyObject.hh"
+#include "GTestXrdHelpers.hh"
+#include "XrdCl/XrdClTaskManager.hh"
+#include "XrdCl/XrdClSIDManager.hh"
+#include "XrdCl/XrdClPropertyList.hh"
+
+//------------------------------------------------------------------------------
+// Declaration
+//------------------------------------------------------------------------------
+
+class A
+{
+  public:
+    A( bool &st ): a(0.0), stat(st) {}
+    ~A() { stat = true; }
+    double  a;
+    bool   &stat;
+};
+
+class B
+{
+  public:
+    int b;
+};
+
+//------------------------------------------------------------------------------
+// UtilityTest class declaration
+//------------------------------------------------------------------------------
+class UtilsTest : public ::testing::Test {};
+
+//------------------------------------------------------------------------------
+// Any test
+//------------------------------------------------------------------------------
+TEST(UtilsTest, AnyTest)
+{
+  bool destructorCalled1 = false;
+  bool destructorCalled2 = false;
+  bool destructorCalled3 = false;
+  A *a1 = new A( destructorCalled1 );
+  A *a2 = new A( destructorCalled2 );
+  A *a3 = new A( destructorCalled3 );
+  A *a4 = 0;
+  B *b  = 0;
+
+  XrdCl::AnyObject *any1 = new XrdCl::AnyObject();
+  XrdCl::AnyObject *any2 = new XrdCl::AnyObject();
+  XrdCl::AnyObject *any3 = new XrdCl::AnyObject();
+  XrdCl::AnyObject *any4 = new XrdCl::AnyObject();
+
+  any1->Set( a1 );
+  any1->Get( b );
+  any1->Get( a4 );
+  EXPECT_TRUE( !b );
+  EXPECT_TRUE( a4 );
+  EXPECT_TRUE( any1->HasOwnership() );
+
+  delete any1;
+  EXPECT_TRUE( destructorCalled1 );
+
+  any2->Set( a2 );
+  any2->Set( (int*)0 );
+  delete any2;
+  EXPECT_TRUE( !destructorCalled2 );
+  delete a2;
+
+  any3->Set( a3, false );
+  EXPECT_TRUE( !any3->HasOwnership() );
+  delete any3;
+  EXPECT_TRUE( !destructorCalled3 );
+  delete a3;
+
+  // test destruction of an empty object
+  delete any4;
+}
+
+//------------------------------------------------------------------------------
+// Some tasks that do something
+//------------------------------------------------------------------------------
+class TestTask1: public XrdCl::Task
+{
+  public:
+    TestTask1( std::vector &runs ): pRuns( runs )
+    {
+      SetName( "TestTask1" );
+    }
+    virtual time_t Run( time_t now )
+    {
+      pRuns.push_back( now );
+      return 0;
+    }
+  private:
+    std::vector &pRuns;
+};
+
+class TestTask2: public XrdCl::Task
+{
+  public:
+    TestTask2( std::vector &runs ): pRuns( runs )
+    {
+      SetName( "TestTask2" );
+    }
+
+    virtual time_t Run( time_t now )
+    {
+      pRuns.push_back( now );
+      if( pRuns.size() >= 5 )
+        return 0;
+      return now+2;
+    }
+  private:
+    std::vector &pRuns;
+};
+
+//------------------------------------------------------------------------------
+// Task Manager test
+//------------------------------------------------------------------------------
+TEST(UtilsTest, TaskManagerTest)
+{
+  using namespace XrdCl;
+
+  std::vector runs1, runs2;
+  Task *tsk1 = new TestTask1( runs1 );
+  Task *tsk2 = new TestTask2( runs2 );
+
+  TaskManager taskMan;
+  EXPECT_TRUE( taskMan.Start() );
+
+  time_t now = ::time(0);
+  taskMan.RegisterTask( tsk1, now+2 );
+  taskMan.RegisterTask( tsk2, now+1 );
+
+  ::sleep( 6 );
+  taskMan.UnregisterTask( tsk2 );
+  ::sleep( 1 );
+
+  EXPECT_EQ( runs1.size(), 1 );
+  EXPECT_EQ( runs2.size(), 3 );
+  EXPECT_TRUE( taskMan.Stop() );
+}
+
+//------------------------------------------------------------------------------
+// SID Manager test
+//------------------------------------------------------------------------------
+TEST(UtilsTest, SIDManagerTest)
+{
+  using namespace XrdCl;
+  std::shared_ptr manager = SIDMgrPool::Instance().GetSIDMgr( "root://fake:1094//dir/file" );
+
+  uint8_t sid1[2];
+  uint8_t sid2[2];
+  uint8_t sid3[2];
+  uint8_t sid4[2];
+  uint8_t sid5[2];
+
+  GTEST_ASSERT_XRDST( manager->AllocateSID( sid1 ) );
+  GTEST_ASSERT_XRDST( manager->AllocateSID( sid2 ) );
+  manager->ReleaseSID( sid2 );
+  GTEST_ASSERT_XRDST( manager->AllocateSID( sid3 ) );
+  GTEST_ASSERT_XRDST( manager->AllocateSID( sid4 ) );
+  GTEST_ASSERT_XRDST( manager->AllocateSID( sid5 ) );
+
+  EXPECT_TRUE( (sid1[0] != sid2[0]) || (sid1[1] != sid2[1]) );
+  EXPECT_EQ( manager->NumberOfTimedOutSIDs(), 0 );
+  manager->TimeOutSID( sid4 );
+  manager->TimeOutSID( sid5 );
+  EXPECT_EQ( manager->NumberOfTimedOutSIDs(), 2 );
+  EXPECT_FALSE( manager->IsTimedOut( sid3 ) );
+  EXPECT_FALSE( manager->IsTimedOut( sid1 ) );
+  EXPECT_TRUE( manager->IsTimedOut( sid4 ) );
+  EXPECT_TRUE( manager->IsTimedOut( sid5 ) );
+  manager->ReleaseTimedOut( sid5 );
+  EXPECT_FALSE( manager->IsTimedOut( sid5 ) );
+  manager->ReleaseAllTimedOut();
+  EXPECT_EQ( manager->NumberOfTimedOutSIDs(), 0 );
+}
+
+//------------------------------------------------------------------------------
+// Property List test
+//------------------------------------------------------------------------------
+TEST(UtilsTest, PropertyListTest)
+{
+  using namespace XrdCl;
+  PropertyList l;
+  l.Set( "s1", "test string 1" );
+  l.Set( "i1", 123456789123ULL );
+
+  uint64_t i1;
+  std::string s1;
+
+  EXPECT_TRUE( l.Get( "s1", s1 ) );
+  EXPECT_EQ( s1, "test string 1" );
+  EXPECT_TRUE( l.Get( "i1", i1 ) );
+  EXPECT_EQ( i1, 123456789123ULL );
+  EXPECT_TRUE( l.HasProperty( "s1" ) );
+  EXPECT_TRUE( !l.HasProperty( "s2" ) );
+  EXPECT_TRUE( l.HasProperty( "i1" ) );
+
+  for( int i = 0; i < 1000; ++i )
+    l.Set( "vect_int", i, i+1000 );
+
+  int i;
+  int num;
+  for( i = 0; l.HasProperty( "vect_int", i ); ++i )
+  {
+    EXPECT_TRUE( l.Get( "vect_int", i, num ) );
+    EXPECT_TRUE( num = i+1000 );
+  }
+  EXPECT_EQ( i, 1000 );
+
+  XRootDStatus st1, st2;
+  st1.SetErrorMessage( "test error message" );
+  l.Set( "status", st1 );
+  EXPECT_TRUE( l.Get( "status", st2 ) );
+  EXPECT_EQ( st2.status, st1.status );
+  EXPECT_EQ( st2.code  , st1.code );
+  EXPECT_EQ( st2.errNo , st1.errNo );
+  EXPECT_EQ( st2.GetErrorMessage(), st1.GetErrorMessage() );
+
+  std::vector v1, v2;
+  v1.push_back( "test string 1" );
+  v1.push_back( "test string 2" );
+  v1.push_back( "test string 3" );
+  l.Set( "vector", v1 );
+  EXPECT_TRUE( l.Get( "vector", v2 ) );
+  for( size_t i = 0; i < v1.size(); ++i )
+    EXPECT_EQ( v1[i], v2[i] );
+}
diff --git a/tests/XrdCl/XrdClZip.cc b/tests/XrdCl/XrdClZip.cc
new file mode 100644
index 00000000000..086d73742d7
--- /dev/null
+++ b/tests/XrdCl/XrdClZip.cc
@@ -0,0 +1,98 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//------------------------------------------------------------------------------
+
+#include "TestEnv.hh"
+#include "Utils.hh"
+#include "IdentityPlugIn.hh"
+
+#include "GTestXrdHelpers.hh"
+#include "XrdZip/XrdZipUtils.hh"
+#include "XrdCl/XrdClZipArchive.hh"
+#include "XrdCl/XrdClZipListHandler.hh"
+#include "XrdCl/XrdClZipOperations.hh"
+
+using namespace XrdClTests;
+using namespace XrdCl;
+
+//------------------------------------------------------------------------------
+// Class declaration
+//------------------------------------------------------------------------------
+class ZipTest: public ::testing::Test {
+  public:
+    void SetUp() override;
+    void TearDown() override;
+    std::string archiveUrl;
+    std::string testFileUrl;
+    ZipArchive zip_file;
+};
+
+void ZipTest::SetUp(){
+  Env *testEnv = TestEnv::GetEnv();
+
+  std::string address;
+  std::string dataPath;
+
+  EXPECT_TRUE( testEnv->GetString( "MainServerURL", address ) );
+  EXPECT_TRUE( testEnv->GetString( "DataPath", dataPath ) );
+
+  std::string path = dataPath + "/data.zip";
+  archiveUrl = address + "/" + path;
+  std::string testFilePath = dataPath + "/san_martino.txt";
+  testFileUrl = address + "/" + testFilePath;
+
+  GTEST_ASSERT_XRDST(WaitFor(OpenArchive(zip_file, archiveUrl, OpenFlags::Read)));
+}
+
+void ZipTest::TearDown()
+{
+  GTEST_ASSERT_XRDST(WaitFor(CloseArchive(zip_file)));
+}
+
+TEST_F(ZipTest, ExtractTest) {
+  /* intentionally empty, just let SetUp() and TearDown() do the work */
+}
+
+TEST_F(ZipTest, OpenFileTest){
+  GTEST_ASSERT_XRDST(zip_file.OpenFile("paper.txt", OpenFlags::Read));
+  // get stat info for the given file
+  StatInfo* info_out;
+  GTEST_ASSERT_XRDST(zip_file.Stat("paper.txt", info_out));
+  GTEST_ASSERT_XRDST(zip_file.CloseFile());
+  GTEST_ASSERT_XRDST_NOTOK(zip_file.OpenFile("gibberish.txt", OpenFlags::Read), errNotFound);
+}
+
+TEST_F(ZipTest, ListFileTest) {
+  DirectoryList* dummy_list;
+  GTEST_ASSERT_XRDST(zip_file.List(dummy_list));
+  EXPECT_TRUE(dummy_list);
+}
+
+TEST_F(ZipTest, GetterTests) {
+  // Get file
+  File* file = NULL;
+  file = &(zip_file.GetFile());
+  EXPECT_TRUE(file);
+
+  // Get checksum
+  uint32_t cksum;
+  GTEST_ASSERT_XRDST(zip_file.GetCRC32("paper.txt", cksum));
+
+  // Get offset (i.e. byte position in the archive)
+  uint64_t offset;
+  GTEST_ASSERT_XRDST(zip_file.GetOffset("paper.txt", offset));
+}
diff --git a/tests/XrdClTests/CMakeLists.txt b/tests/XrdClTests/CMakeLists.txt
index b1603333c5e..238e2509899 100644
--- a/tests/XrdClTests/CMakeLists.txt
+++ b/tests/XrdClTests/CMakeLists.txt
@@ -1,7 +1,4 @@
 
-include( XRootDCommon )
-include_directories( ${CPPUNIT_INCLUDE_DIRS} ../common)
-
 add_subdirectory( tls )
 
 set( LIB_XRD_CL_TEST_MONITOR XrdClTestMonitor-${PLUGIN_VERSION} )
@@ -35,6 +32,8 @@ target_link_libraries(
   ZLIB::ZLIB
   XrdCl )
 
+target_include_directories( XrdClTests PRIVATE ../common ${CPPUNIT_INCLUDE_DIRS} )
+
 add_library(
   ${LIB_XRD_CL_TEST_MONITOR} MODULE
   MonitorTestLib.cc
@@ -45,6 +44,8 @@ target_link_libraries(
   XrdClTestsHelper
   XrdCl )
 
+target_include_directories( ${LIB_XRD_CL_TEST_MONITOR} PRIVATE ../common )
+
 foreach(TEST_SUITE
     # File
     # FileCopy
@@ -59,6 +60,7 @@ foreach(TEST_SUITE
 )
   add_test(NAME XrdCl::${TEST_SUITE}
     COMMAND $ $ "All Tests/${TEST_SUITE}Test")
+  set_tests_properties(XrdCl::${TEST_SUITE} PROPERTIES RUN_SERIAL TRUE)
 endforeach()
 
 #-------------------------------------------------------------------------------
diff --git a/tests/XrdClTests/FileCopyTest.cc b/tests/XrdClTests/FileCopyTest.cc
index 5c9c8de4d57..90e6845e795 100644
--- a/tests/XrdClTests/FileCopyTest.cc
+++ b/tests/XrdClTests/FileCopyTest.cc
@@ -397,7 +397,7 @@ void FileCopyTest::CopyTestFunc( bool thirdParty )
     CPPUNIT_ASSERT_XRDST_NOTOK( process12.Run(0), XrdCl::errCheckSumError );
     XrdCl::StatInfo *info = 0;
     XrdCl::XRootDStatus status = fs.Stat( targetPath, info );
-    CPPUNIT_ASSERT_XRDST( status.status == XrdCl::stError && st.code == XrdCl::errNotFound );
+    CPPUNIT_ASSERT_XRDST( status.status == XrdCl::stError && status.code == XrdCl::errNotFound );
     properties.Clear();
 
     //--------------------------------------------------------------------------
diff --git a/tests/XrdClTests/LocalFileHandlerTest.cc b/tests/XrdClTests/LocalFileHandlerTest.cc
index 0a58ed6effd..f8618c14f7b 100644
--- a/tests/XrdClTests/LocalFileHandlerTest.cc
+++ b/tests/XrdClTests/LocalFileHandlerTest.cc
@@ -142,7 +142,7 @@ void LocalFileHandlerTest::WriteTest(){
    //----------------------------------------------------------------------------
    // Read file with POSIX calls to confirm correct write
    //----------------------------------------------------------------------------
-   int fd = open( targetURL.c_str(), flags );
+   int fd = open( targetURL.c_str(), O_RDWR );
    int rc = read( fd, buffer, int( writeSize ) );
    CPPUNIT_ASSERT_EQUAL( rc, int( writeSize ) );
    std::string read( (char *)buffer, writeSize );
@@ -176,7 +176,7 @@ void LocalFileHandlerTest::WriteWithOffsetTest(){
    //----------------------------------------------------------------------------
    // Read file with POSIX calls to confirm correct write
    //----------------------------------------------------------------------------
-   int fd = open( targetURL.c_str(), flags );
+   int fd = open( targetURL.c_str(), O_RDWR );
    int rc = read( fd, buffer, offset );
    CPPUNIT_ASSERT_EQUAL( rc, int( offset ) );
    std::string read( (char *)buffer, offset );
@@ -207,7 +207,7 @@ void LocalFileHandlerTest::WriteMkdirTest(){
    //----------------------------------------------------------------------------
    // Read file with POSIX calls to confirm correct write
    //----------------------------------------------------------------------------
-   int fd = open( targetURL.c_str(), flags );
+   int fd = open( targetURL.c_str(), O_RDWR );
    int rc = read( fd, buffer, writeSize );
    CPPUNIT_ASSERT_EQUAL( rc, int( writeSize ) );
    std::string read( buffer, writeSize );
diff --git a/tests/XrdClTests/PostMasterTest.cc b/tests/XrdClTests/PostMasterTest.cc
index e8e4b02c337..e54a696c841 100644
--- a/tests/XrdClTests/PostMasterTest.cc
+++ b/tests/XrdClTests/PostMasterTest.cc
@@ -56,23 +56,22 @@ CPPUNIT_TEST_SUITE_REGISTRATION( PostMasterTest );
 //------------------------------------------------------------------------------
 namespace
 {
-  class PostMasterFinalizer
+  class PostMasterFetch
   {
     public:
-      PostMasterFinalizer( XrdCl::PostMaster *pm = 0 ): pPostMaster(pm) {}
-      ~PostMasterFinalizer()
-      {
-        if( pPostMaster )
-        {
-          pPostMaster->Stop();
-          pPostMaster->Finalize();
-        }
+      PostMasterFetch() { }
+      ~PostMasterFetch()  { }
+      XrdCl::PostMaster *Get() {
+        return XrdCl::DefaultEnv::GetPostMaster();
+      }
+      XrdCl::PostMaster *Reset() {
+        XrdCl::PostMaster *pm = Get();
+        pm->Stop();
+        pm->Finalize();
+        CPPUNIT_ASSERT( pm->Initialize() != 0 );
+        CPPUNIT_ASSERT( pm->Start() != 0 );
+        return pm;
       }
-      void Set( XrdCl::PostMaster *pm ) { pPostMaster = pm; }
-      XrdCl::PostMaster *Get() { return pPostMaster; }
-
-    private:
-      XrdCl::PostMaster *pPostMaster;
   };
 }
 
@@ -285,17 +284,15 @@ void *TestThreadFunc( void *arg )
 void PostMasterTest::ThreadingTest()
 {
   using namespace XrdCl;
-  PostMaster postMaster;
-  PostMasterFinalizer finalizer( &postMaster );
-  postMaster.Initialize();
-  postMaster.Start();
+  PostMasterFetch pmfetch;
+  PostMaster *postMaster = pmfetch.Get();
 
   pthread_t thread[100];
   ArgHelper helper[100];
 
   for( int i = 0; i < 100; ++i )
   {
-    helper[i].pm    = &postMaster;
+    helper[i].pm    = postMaster;
     helper[i].index = i;
     pthread_create( &thread[i], 0, TestThreadFunc, &helper[i] );
   }
@@ -319,10 +316,8 @@ void PostMasterTest::FunctionalTest()
   env->PutInt( "TimeoutResolution", 1 );
   env->PutInt( "ConnectionWindow", 15 );
 
-  PostMaster postMaster;
-  PostMasterFinalizer finalizer( &postMaster );
-  postMaster.Initialize();
-  postMaster.Start();
+  PostMasterFetch pmfetch;
+  PostMaster *postMaster = pmfetch.Get();
 
   std::string address;
   CPPUNIT_ASSERT( testEnv->GetString( "MainServerURL", address ) );
@@ -348,7 +343,7 @@ void PostMasterTest::FunctionalTest()
   request->dlen        = 0;
   XRootDTransport::MarshallRequest( &m1 );
 
-  CPPUNIT_ASSERT_XRDST( postMaster.Send( host, &m1, &msgHandler1, false, expires ) );
+  CPPUNIT_ASSERT_XRDST( postMaster->Send( host, &m1, &msgHandler1, false, expires ) );
 
   CPPUNIT_ASSERT_XRDST( msgHandler1.WaitFor( m2 ) );
   ServerResponse *resp = (ServerResponse *)m2.GetBuffer();
@@ -367,14 +362,14 @@ void PostMasterTest::FunctionalTest()
   msgHandler2.SetFilter( 1, 2 );
   time_t shortexp = ::time(0) + 3;
   msgHandler2.SetExpiration( shortexp );
-  CPPUNIT_ASSERT_XRDST( postMaster.Send( localhost1, &m1, &msgHandler2, false,
+  CPPUNIT_ASSERT_XRDST( postMaster->Send( localhost1, &m1, &msgHandler2, false,
                         shortexp ) );
   CPPUNIT_ASSERT_XRDST_NOTOK( msgHandler2.WaitFor( m2 ), errOperationExpired );
 
   SyncMsgHandler msgHandler3;
   msgHandler3.SetFilter( 1, 2 );
   msgHandler3.SetExpiration( expires );
-  CPPUNIT_ASSERT_XRDST( postMaster.Send( localhost1, &m1, &msgHandler3, false,
+  CPPUNIT_ASSERT_XRDST( postMaster->Send( localhost1, &m1, &msgHandler3, false,
                                           expires ) );
   CPPUNIT_ASSERT_XRDST_NOTOK( msgHandler3.WaitFor( m2 ), errConnectionError );
 
@@ -385,7 +380,7 @@ void PostMasterTest::FunctionalTest()
   Status st1, st2;
   const char *name   = 0;
 
-  CPPUNIT_ASSERT_XRDST( postMaster.QueryTransport( host,
+  CPPUNIT_ASSERT_XRDST( postMaster->QueryTransport( host,
                                                    TransportQuery::Name,
                                                    nameObj ) );
   nameObj.Get( name );
@@ -393,15 +388,11 @@ void PostMasterTest::FunctionalTest()
   CPPUNIT_ASSERT( name );
   CPPUNIT_ASSERT( !::strcmp( name, "XRootD" ) );
 
-  postMaster.Stop();
-  postMaster.Finalize();
-
   //----------------------------------------------------------------------------
   // Reinitialize and try to do something
   //----------------------------------------------------------------------------
   env->PutInt( "LoadBalancerTTL", 5 );
-  postMaster.Initialize();
-  postMaster.Start();
+  postMaster = pmfetch.Reset();
 
   m2.Free();
   m1.Zero();
@@ -416,7 +407,7 @@ void PostMasterTest::FunctionalTest()
   SyncMsgHandler msgHandler4;
   msgHandler4.SetFilter( 1, 2 );
   msgHandler4.SetExpiration( expires );
-  CPPUNIT_ASSERT_XRDST( postMaster.Send( host, &m1, &msgHandler4, false, expires ) );
+  CPPUNIT_ASSERT_XRDST( postMaster->Send( host, &m1, &msgHandler4, false, expires ) );
 
   CPPUNIT_ASSERT_XRDST( msgHandler4.WaitFor( m2 ) );
   resp = (ServerResponse *)m2.GetBuffer();
@@ -432,7 +423,7 @@ void PostMasterTest::FunctionalTest()
   SyncMsgHandler msgHandler5;
   msgHandler5.SetFilter( 1, 2 );
   msgHandler5.SetExpiration( expires );
-  CPPUNIT_ASSERT_XRDST( postMaster.Send( host, &m1, &msgHandler5, false, expires ) );
+  CPPUNIT_ASSERT_XRDST( postMaster->Send( host, &m1, &msgHandler5, false, expires ) );
 
   CPPUNIT_ASSERT_XRDST( msgHandler5.WaitFor( m2 ) );
   resp = (ServerResponse *)m2.GetBuffer();
@@ -452,9 +443,8 @@ void PostMasterTest::PingIPv6()
   //----------------------------------------------------------------------------
   // Initialize the stuff
   //----------------------------------------------------------------------------
-  PostMaster postMaster;
-  postMaster.Initialize();
-  postMaster.Start();
+  PostMasterFetch pmfetch;
+  PostMaster *postMaster = pmfetch.Get();
 
   //----------------------------------------------------------------------------
   // Build the message
@@ -479,10 +469,10 @@ void PostMasterTest::PingIPv6()
   //----------------------------------------------------------------------------
   // Send the message - localhost1
   //----------------------------------------------------------------------------
-  sc = postMaster.Send( localhost1, &m1, false, 1200 );
+  sc = postMaster->Send( localhost1, &m1, false, 1200 );
   CPPUNIT_ASSERT( sc.IsOK() );
 
-  sc = postMaster.Receive( localhost1, m2, &f1, false, 1200 );
+  sc = postMaster->Receive( localhost1, m2, &f1, false, 1200 );
   CPPUNIT_ASSERT( sc.IsOK() );
   ServerResponse *resp = (ServerResponse *)m2->GetBuffer();
   CPPUNIT_ASSERT( resp != 0 );
@@ -492,21 +482,15 @@ void PostMasterTest::PingIPv6()
   //----------------------------------------------------------------------------
   // Send the message - localhost2
   //----------------------------------------------------------------------------
-  sc = postMaster.Send( localhost2, &m1, false, 1200 );
+  sc = postMaster->Send( localhost2, &m1, false, 1200 );
   CPPUNIT_ASSERT( sc.IsOK() );
 
-  sc = postMaster.Receive( localhost2, m2, &f1, 1200 );
+  sc = postMaster->Receive( localhost2, m2, &f1, 1200 );
   CPPUNIT_ASSERT( sc.IsOK() );
   resp = (ServerResponse *)m2->GetBuffer();
   CPPUNIT_ASSERT( resp != 0 );
   CPPUNIT_ASSERT( resp->hdr.status == kXR_ok );
   CPPUNIT_ASSERT( m2->GetSize() == 8 );
-
-  //----------------------------------------------------------------------------
-  // Clean up
-  //----------------------------------------------------------------------------
-  postMaster.Stop();
-  postMaster.Finalize();
 #endif
 }
 
@@ -547,10 +531,8 @@ void PostMasterTest::MultiIPConnectionTest()
   env->PutInt( "TimeoutResolution", 1 );
   env->PutInt( "ConnectionWindow",  5 );
 
-  PostMaster postMaster;
-  PostMasterFinalizer finalizer( &postMaster );
-  postMaster.Initialize();
-  postMaster.Start();
+  PostMasterFetch pmfetch;
+  PostMaster *postMaster = pmfetch.Get();
 
   std::string address;
   CPPUNIT_ASSERT( testEnv->GetString( "MultiIPServerURL", address ) );
@@ -569,7 +551,7 @@ void PostMasterTest::MultiIPConnectionTest()
   msgHandler1.SetFilter( 1, 2 );
   msgHandler1.SetExpiration( expires );
   Message *m = CreatePing( 1, 2 );
-  CPPUNIT_ASSERT_XRDST_NOTOK( postMaster.Send( url1, m, &msgHandler1, false, expires ),
+  CPPUNIT_ASSERT_XRDST_NOTOK( postMaster->Send( url1, m, &msgHandler1, false, expires ),
                               errInvalidAddr );
 
   //----------------------------------------------------------------------------
@@ -580,7 +562,7 @@ void PostMasterTest::MultiIPConnectionTest()
   msgHandler2.SetExpiration( expires );
   Message m2;
 
-  CPPUNIT_ASSERT_XRDST( postMaster.Send( url2, m, &msgHandler2, false, expires ) );
+  CPPUNIT_ASSERT_XRDST( postMaster->Send( url2, m, &msgHandler2, false, expires ) );
   CPPUNIT_ASSERT_XRDST_NOTOK( msgHandler2.WaitFor( m2 ), errConnectionError );
 
   //----------------------------------------------------------------------------
@@ -590,7 +572,7 @@ void PostMasterTest::MultiIPConnectionTest()
   msgHandler3.SetFilter( 1, 2 );
   msgHandler3.SetExpiration( expires );
 
-  CPPUNIT_ASSERT_XRDST( postMaster.Send( url3, m, &msgHandler3, false, expires ) );
+  CPPUNIT_ASSERT_XRDST( postMaster->Send( url3, m, &msgHandler3, false, expires ) );
   CPPUNIT_ASSERT_XRDST( msgHandler3.WaitFor( m2 ) );
   ServerResponse *resp = (ServerResponse *)m2.GetBuffer();
   CPPUNIT_ASSERT( resp != 0 );
diff --git a/tests/XrdClTests/UtilsTest.cc b/tests/XrdClTests/UtilsTest.cc
index 86104bc57f7..7bec1130f41 100644
--- a/tests/XrdClTests/UtilsTest.cc
+++ b/tests/XrdClTests/UtilsTest.cc
@@ -212,7 +212,7 @@ void UtilsTest::SIDManagerTest()
 }
 
 //------------------------------------------------------------------------------
-// SID Manager test
+// Property List test
 //------------------------------------------------------------------------------
 void UtilsTest::PropertyListTest()
 {
diff --git a/tests/XrdClTests/tls/CMakeLists.txt b/tests/XrdClTests/tls/CMakeLists.txt
index 7ab152723c0..93b3e145a10 100644
--- a/tests/XrdClTests/tls/CMakeLists.txt
+++ b/tests/XrdClTests/tls/CMakeLists.txt
@@ -1,5 +1,4 @@
 
-include( XRootDCommon )
 
 #-------------------------------------------------------------------------------
 # xrdcopy
diff --git a/tests/XrdEc/CMakeLists.txt b/tests/XrdEc/CMakeLists.txt
new file mode 100644
index 00000000000..0531c35ec43
--- /dev/null
+++ b/tests/XrdEc/CMakeLists.txt
@@ -0,0 +1,25 @@
+add_executable(xrdec-unit-tests
+  MicroTest.cc
+  ../common/Server.cc
+  ../common/Utils.cc
+  ../common/TestEnv.cc
+)
+
+target_link_libraries(xrdec-unit-tests
+  XrdEc
+  XrdCl
+  XrdXml
+  XrdUtils
+  ZLIB::ZLIB
+  GTest::GTest
+  GTest::Main
+  ${ISAL_LIBRARIES}
+)
+
+target_include_directories(xrdec-unit-tests
+  PRIVATE ${CMAKE_SOURCE_DIR}/src
+  PRIVATE ../common
+  ${ISAL_INCLUDE_DIRS}
+)
+
+gtest_discover_tests(xrdec-unit-tests TEST_PREFIX XrdCl::)
diff --git a/tests/XrdEc/MicroTest.cc b/tests/XrdEc/MicroTest.cc
new file mode 100644
index 00000000000..b39b6d24558
--- /dev/null
+++ b/tests/XrdEc/MicroTest.cc
@@ -0,0 +1,749 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2023 by European Organization for Nuclear Research (CERN)
+// Author: Angelo Galavotti 
+//------------------------------------------------------------------------------
+// This file is part of the XRootD software suite.
+//
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see .
+//
+// In applying this licence, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+//------------------------------------------------------------------------------
+
+#include "TestEnv.hh"
+#include "../XrdCl/GTestXrdHelpers.hh"
+#include 
+
+#include "XrdEc/XrdEcStrmWriter.hh"
+#include "XrdEc/XrdEcReader.hh"
+#include "XrdEc/XrdEcObjCfg.hh"
+
+#include "XrdCl/XrdClMessageUtils.hh"
+
+#include "XrdSys/XrdSysPlatform.hh"
+
+#include "XrdZip/XrdZipCDFH.hh"
+
+#include 
+#include 
+#include 
+
+#include 
+#include 
+#include 
+#include 
+#include 
+
+using namespace XrdEc;
+
+//------------------------------------------------------------------------------
+// Declaration
+//------------------------------------------------------------------------------
+class XrdEcTests : public ::testing::Test
+{
+  public:
+    void Init( bool usecrc32c );
+
+    inline void AlignedWriteTestImpl( bool usecrc32c )
+    {
+      // create the data and stripe directories
+      Init( usecrc32c );
+      // run the test
+      AlignedWriteRaw();
+      // verify that we wrote the data correctly
+      Verify();
+      // clean up the data directory
+      CleanUp();
+    }
+
+    inline void AlignedWriteTest()
+    {
+      AlignedWriteTestImpl( true );
+    }
+
+    inline void AlignedWriteTestIsalCrcNoMt()
+    {
+      AlignedWriteTestImpl( false );
+    }
+
+    inline void VectorReadTest(){
+	Init(true);
+
+	AlignedWriteRaw();
+
+	Verify();
+
+	uint32_t seed = std::chrono::system_clock::now().time_since_epoch().count();
+
+	VerifyVectorRead(seed);
+
+	CleanUp();
+    }
+
+    inline void IllegalVectorReadTest(){
+		Init(true);
+
+		AlignedWriteRaw();
+
+		Verify();
+
+		uint32_t seed =
+				std::chrono::system_clock::now().time_since_epoch().count();
+
+		IllegalVectorRead(seed);
+
+		CleanUp();
+    }
+
+    inline void AlignedWrite1MissingTestImpl( bool usecrc32c )
+    {
+      // initialize directories
+      Init( usecrc32c );
+      UrlNotReachable( 2 );
+      // run the test
+      AlignedWriteRaw();
+      // verify that we wrote the data correctly
+      Verify();
+      // clean up
+      UrlReachable( 2 );
+      CleanUp();
+    }
+
+    inline void AlignedWrite1MissingTest()
+    {
+      AlignedWrite1MissingTestImpl( true );
+    }
+
+    inline void AlignedWrite1MissingTestIsalCrcNoMt()
+    {
+      AlignedWrite1MissingTestImpl( false );
+    }
+
+    inline void AlignedWrite2MissingTestImpl( bool usecrc32c )
+    {
+      // initialize directories
+      Init( usecrc32c );
+      UrlNotReachable( 2 );
+      UrlNotReachable( 3 );
+      // run the test
+      AlignedWriteRaw();
+      // verify that we wrote the data correctly
+      Verify();
+      // clean up
+      UrlReachable( 2 );
+      UrlReachable( 3 );
+      CleanUp();
+    }
+
+    inline void AlignedWrite2MissingTest()
+    {
+      AlignedWrite2MissingTestImpl( true );
+    }
+
+    inline void AlignedWrite2MissingTestIsalCrcNoMt()
+    {
+      AlignedWrite2MissingTestImpl( false );
+    }
+
+    void VarlenWriteTest( uint32_t wrtlen, bool usecrc32c );
+
+    inline void SmallWriteTest()
+    {
+      VarlenWriteTest( 7, true );
+    }
+
+    inline void SmallWriteTestIsalCrcNoMt()
+    {
+      VarlenWriteTest( 7, false );
+    }
+
+    void BigWriteTest()
+    {
+      VarlenWriteTest( 77, true );
+    }
+
+    void BigWriteTestIsalCrcNoMt()
+    {
+      VarlenWriteTest( 77, false );
+    }
+
+    void Verify()
+    {
+      ReadVerifyAll();
+      CorruptedReadVerify();
+    }
+
+    void VerifyVectorRead(uint32_t randomSeed);
+
+    void IllegalVectorRead(uint32_t randomSeed);
+
+    void CleanUp();
+
+    inline void ReadVerifyAll()
+    {
+      AlignedReadVerify();
+      PastEndReadVerify();
+      SmallChunkReadVerify();
+      BigChunkReadVerify();
+
+      for( size_t i = 0; i < 10; ++i )
+        RandomReadVerify();
+    }
+
+    void ReadVerify( uint32_t rdsize, uint64_t maxrd = std::numeric_limits::max() );
+
+    void RandomReadVerify();
+
+    void Corrupted1stBlkReadVerify();
+
+    inline void AlignedReadVerify()
+    {
+      ReadVerify( chsize, rawdata.size() );
+    }
+
+    inline void PastEndReadVerify()
+    {
+      ReadVerify( chsize );
+    }
+
+    inline void SmallChunkReadVerify()
+    {
+      ReadVerify( 5 );
+    }
+
+    inline void BigChunkReadVerify()
+    {
+      ReadVerify( 23 );
+    }
+
+    void CorruptedReadVerify();
+
+    void CorruptChunk( size_t blknb, size_t strpnb );
+
+    void UrlNotReachable( size_t index );
+    void UrlReachable( size_t index );
+
+  private:
+
+    void AlignedWriteRaw();
+
+    void copy_rawdata( char *buffer, size_t size )
+    {
+      const char *begin = buffer;
+      const char *end   = begin + size;
+      std::copy( begin, end, std::back_inserter( rawdata ) );
+    }
+
+    std::string datadir;
+    std::unique_ptr objcfg;
+
+    static const size_t nbdata   = 4;
+    static const size_t nbparity = 2;
+    static const size_t chsize   = 16;
+    static const size_t nbiters  = 16;
+
+    static const size_t lfhsize  = 30;
+
+    std::vector rawdata;
+};
+
+TEST_F(XrdEcTests, AlignedWriteTest)
+{
+  AlignedWriteTest();
+}
+
+TEST_F(XrdEcTests, SmallWriteTest)
+{
+  SmallWriteTest();
+}
+
+TEST_F(XrdEcTests, BigWriteTest)
+{
+  BigWriteTest();
+}
+
+TEST_F(XrdEcTests, VectorReadTest)
+{
+  VectorReadTest();
+}
+
+TEST_F(XrdEcTests, IllegalVectorReadTest)
+{
+  IllegalVectorReadTest();
+}
+
+TEST_F(XrdEcTests, AlignedWrite1MissingTest)
+{
+  AlignedWrite1MissingTest();
+}
+
+TEST_F(XrdEcTests, AlignedWrite2MissingTest) 
+{
+  AlignedWrite2MissingTest();
+}
+
+TEST_F(XrdEcTests, AlignedWriteTestIsalCrcNoMt)
+{
+  AlignedWriteTestIsalCrcNoMt();
+}
+
+TEST_F(XrdEcTests, SmallWriteTestIsalCrcNoMt)
+{
+  SmallWriteTestIsalCrcNoMt();
+}
+
+TEST_F(XrdEcTests, BigWriteTestIsalCrcNoMt)
+{
+  BigWriteTestIsalCrcNoMt();
+}
+
+TEST_F(XrdEcTests, AlignedWrite1MissingTestIsalCrcNoMt)
+{
+  AlignedWrite1MissingTestIsalCrcNoMt();
+}
+
+TEST_F(XrdEcTests, AlignedWrite2MissingTestIsalCrcNoMt)
+{
+  AlignedWrite2MissingTestIsalCrcNoMt();
+}
+
+
+void XrdEcTests::Init( bool usecrc32c )
+{
+  objcfg.reset( new ObjCfg( "test.txt", nbdata, nbparity, chsize, usecrc32c, true ) );
+  rawdata.clear();
+
+  char tmpdir[MAXPATHLEN];
+  EXPECT_TRUE( getcwd(tmpdir, MAXPATHLEN - 21) );
+  strcat(tmpdir, "/xrootd-xrdec-XXXXXX");
+  // create the data directory
+  EXPECT_TRUE( mkdtemp(tmpdir) );
+  datadir = tmpdir;
+  // create a directory for each stripe
+  size_t nbstrps = objcfg->nbdata + 2 * objcfg->nbparity;
+  for( size_t i = 0; i < nbstrps; ++i )
+  {
+    std::stringstream ss;
+    ss << std::setfill('0') << std::setw( 2 ) << i;
+    std::string strp = datadir + '/' + ss.str() + '/';
+    objcfg->plgr.emplace_back( strp );
+    EXPECT_EQ( mkdir( strp.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH ), 0 );
+  }
+}
+
+void XrdEcTests::CorruptChunk( size_t blknb, size_t strpnb )
+{
+  Reader reader( *objcfg );
+  // open the data object
+  XrdCl::SyncResponseHandler handler1;
+  reader.Open( &handler1 );
+  handler1.WaitForResponse();
+  XrdCl::XRootDStatus *status = handler1.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+
+  // get the CD buffer
+  std::string fn     = objcfg->GetFileName( blknb, strpnb );
+  std::string url    = reader.urlmap[fn];
+  buffer_t    cdbuff = reader.dataarchs[url]->GetCD();
+
+  // close the data object
+  XrdCl::SyncResponseHandler handler2;
+  reader.Close( &handler2 );
+  handler2.WaitForResponse();
+  status = handler2.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+
+  // parse the CD buffer
+  const char *buff   = cdbuff.data();
+  size_t      size   = cdbuff.size();
+  XrdZip::cdvec_t cdvec;
+  XrdZip::cdmap_t cdmap;
+  std::tie(cdvec, cdmap ) = XrdZip::CDFH::Parse( buff, size );
+
+  // now corrupt the chunk (put wrong checksum)
+  XrdZip::CDFH &cdfh = *cdvec[cdmap[fn]];
+  uint64_t offset = cdfh.offset + lfhsize + fn.size(); // offset of the data
+  XrdCl::File f;
+  GTEST_ASSERT_XRDST( f.Open( url, XrdCl::OpenFlags::Write ) );
+  std::string str = "XXXXXXXX";
+  GTEST_ASSERT_XRDST( f.Write( offset, str.size(), str.c_str() ) );
+  GTEST_ASSERT_XRDST( f.Close() );
+}
+
+void XrdEcTests::UrlNotReachable( size_t index )
+{
+  XrdCl::URL url( objcfg->plgr[index] );
+  EXPECT_EQ( chmod( url.GetPath().c_str(), 0 ), 0 );
+}
+
+void XrdEcTests::UrlReachable( size_t index )
+{
+  XrdCl::URL url( objcfg->plgr[index] );
+  mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH |
+                S_IXUSR | S_IXGRP | S_IXOTH;
+  EXPECT_EQ( chmod( url.GetPath().c_str(), mode ), 0 );
+}
+
+void XrdEcTests::CorruptedReadVerify()
+{
+  UrlNotReachable( 0 );
+  ReadVerifyAll();
+  UrlNotReachable( 1 );
+  ReadVerifyAll();
+  UrlReachable( 0 );
+  UrlReachable( 1 );
+
+  CorruptChunk( 0, 1 );
+  ReadVerifyAll();
+
+  CorruptChunk( 0, 2 );
+  ReadVerifyAll();
+
+}
+
+void XrdEcTests::VerifyVectorRead(uint32_t seed){
+	  Reader reader( *objcfg );
+	  // open the data object
+	  XrdCl::SyncResponseHandler handler1;
+	  reader.Open( &handler1 );
+	  handler1.WaitForResponse();
+	  XrdCl::XRootDStatus *status = handler1.GetStatus();
+	  GTEST_ASSERT_XRDST( *status );
+	  delete status;
+
+	  std::default_random_engine random_engine(seed);
+
+	  std::vector> buffers(5);
+	  std::vector expected;
+	  XrdCl::ChunkList chunks;
+	  for(int i = 0; i < 5; i++){
+		  std::uniform_int_distribution sizeGen(0, rawdata.size()/4);
+		  uint32_t size = sizeGen(random_engine);
+		  std::uniform_int_distribution offsetGen(0, rawdata.size() - size);
+		  uint32_t offset = offsetGen(random_engine);
+
+		  buffers[i].resize(size);
+		  chunks.push_back(XrdCl::ChunkInfo(offset, size, buffers[i].data()));
+
+		  std::string resultExp( rawdata.data() + offset, size );
+		  expected.push_back(resultExp);
+	  }
+
+	  XrdCl::SyncResponseHandler h;
+	  reader.VectorRead(chunks, nullptr, &h, 0);
+	    h.WaitForResponse();
+	    status = h.GetStatus();
+	    GTEST_ASSERT_XRDST( *status );
+	    delete status;
+	    for(int i = 0; i < 5; i++){
+		std::string result(buffers[i].data(), expected[i].size());
+		EXPECT_EQ( result, expected[i] );
+	    }
+
+	    XrdCl::SyncResponseHandler handler2;
+	      reader.Close( &handler2 );
+	      handler2.WaitForResponse();
+	      status = handler2.GetStatus();
+	      GTEST_ASSERT_XRDST( *status );
+	      delete status;
+}
+
+void XrdEcTests::IllegalVectorRead(uint32_t seed){
+	Reader reader(*objcfg);
+	// open the data object
+	XrdCl::SyncResponseHandler handler1;
+	reader.Open(&handler1);
+	handler1.WaitForResponse();
+	XrdCl::XRootDStatus *status = handler1.GetStatus();
+	GTEST_ASSERT_XRDST( *status );
+	delete status;
+
+	std::default_random_engine random_engine(seed);
+
+	std::vector> buffers(5);
+	XrdCl::ChunkList chunks;
+	for (int i = 0; i < 5; i++)
+	{
+		std::uniform_int_distribution sizeGen(1, rawdata.size() / 4);
+		uint32_t size = sizeGen(random_engine);
+		std::uniform_int_distribution offsetGen(0,
+				rawdata.size() - size);
+		uint32_t offset = offsetGen(random_engine);
+		if (i == 0)
+			offset = rawdata.size() - size / 2;
+
+		buffers[i].resize(size);
+
+		chunks.push_back(XrdCl::ChunkInfo(offset, size, buffers[i].data()));
+
+	}
+
+	XrdCl::SyncResponseHandler h;
+	reader.VectorRead(chunks, nullptr, &h, 0);
+	h.WaitForResponse();
+	status = h.GetStatus();
+	// the response should be negative since one of the reads was over the file end
+	EXPECT_FALSE(status->IsOK());
+	delete status;
+
+	buffers.clear();
+	buffers.resize(1025);
+	chunks.clear();
+	for (int i = 0; i < 1025; i++)
+	{
+		std::uniform_int_distribution sizeGen(1, rawdata.size() / 4);
+		uint32_t size = sizeGen(random_engine);
+		std::uniform_int_distribution offsetGen(0,
+				rawdata.size() - size);
+		uint32_t offset = offsetGen(random_engine);
+
+		buffers[i].resize(size);
+
+		chunks.push_back(XrdCl::ChunkInfo(offset, size, buffers[i].data()));
+
+	}
+
+	XrdCl::SyncResponseHandler h2;
+	reader.VectorRead(chunks, nullptr, &h2, 0);
+	h2.WaitForResponse();
+	status = h2.GetStatus();
+	// the response should be negative since we requested too many reads
+	EXPECT_FALSE(status->IsOK());
+	delete status;
+
+	XrdCl::SyncResponseHandler handler2;
+	reader.Close(&handler2);
+	handler2.WaitForResponse();
+	status = handler2.GetStatus();
+	GTEST_ASSERT_XRDST( *status );
+	delete status;
+}
+
+void XrdEcTests::ReadVerify( uint32_t rdsize, uint64_t maxrd )
+{
+  Reader reader( *objcfg );
+  // open the data object
+  XrdCl::SyncResponseHandler handler1;
+  reader.Open( &handler1 );
+  handler1.WaitForResponse();
+  XrdCl::XRootDStatus *status = handler1.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+  
+  uint64_t  rdoff  = 0;
+  char     *rdbuff = new char[rdsize]; 
+  uint32_t  bytesrd = 0;
+  uint64_t  total_bytesrd = 0;
+  do
+  {
+    XrdCl::SyncResponseHandler h;
+    reader.Read( rdoff, rdsize, rdbuff, &h, 0 );
+    h.WaitForResponse();
+    status = h.GetStatus();
+    GTEST_ASSERT_XRDST( *status );
+    // get the actual result
+    auto rsp = h.GetResponse();
+    XrdCl::ChunkInfo *ch = nullptr;
+    rsp->Get( ch );
+    bytesrd = ch->length;
+    std::string result( reinterpret_cast( ch->buffer ), bytesrd );
+    // get the expected result
+    size_t rawoff = rdoff;
+    size_t rawsz  = rdsize;
+    if( rawoff + rawsz > rawdata.size() ) rawsz = rawdata.size() - rawoff;
+    std::string expected( rawdata.data() + rawoff, rawsz );
+    // make sure the expected and actual results are the same
+    EXPECT_EQ( result, expected );
+    delete status;
+    delete rsp;
+    rdoff += bytesrd;
+    total_bytesrd += bytesrd;
+  }
+  while( bytesrd == rdsize && total_bytesrd < maxrd );
+  delete[] rdbuff;
+ 
+  // close the data object
+  XrdCl::SyncResponseHandler handler2;
+  reader.Close( &handler2 );
+  handler2.WaitForResponse();
+  status = handler2.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+}
+
+void XrdEcTests::RandomReadVerify()
+{
+  size_t filesize = rawdata.size();
+  static std::default_random_engine random_engine( std::chrono::system_clock::now().time_since_epoch().count() );
+  std::uniform_int_distribution offdistr( 0, filesize );
+  uint64_t rdoff = offdistr( random_engine );
+  std::uniform_int_distribution lendistr( rdoff, filesize + 32 );
+  uint32_t rdlen = lendistr( random_engine );
+
+  Reader reader( *objcfg );
+  // open the data object
+  XrdCl::SyncResponseHandler handler1;
+  reader.Open( &handler1 );
+  handler1.WaitForResponse();
+  XrdCl::XRootDStatus *status = handler1.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+
+  // read the data
+  char *rdbuff = new char[rdlen];
+  XrdCl::SyncResponseHandler h;
+  reader.Read( rdoff, rdlen, rdbuff, &h, 0 );
+  h.WaitForResponse();
+  status = h.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  // get the actual result
+  auto rsp = h.GetResponse();
+  XrdCl::ChunkInfo *ch = nullptr;
+  rsp->Get( ch );
+  uint32_t bytesrd = ch->length;
+  std::string result( reinterpret_cast( ch->buffer ), bytesrd );
+  // get the expected result
+  size_t rawoff = rdoff;
+  size_t rawlen  = rdlen;
+  if( rawoff > rawdata.size() ) rawlen = 0;
+  else if( rawoff + rawlen > rawdata.size() ) rawlen = rawdata.size() - rawoff;
+  std::string expected( rawdata.data() + rawoff, rawlen );
+  // make sure the expected and actual results are the same
+  EXPECT_EQ( result, expected );
+  delete status;
+  delete rsp;
+  delete[] rdbuff;
+
+  // close the data object
+  XrdCl::SyncResponseHandler handler2;
+  reader.Close( &handler2 );
+  handler2.WaitForResponse();
+  status = handler2.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+}
+
+void XrdEcTests::Corrupted1stBlkReadVerify()
+{
+  uint64_t rdoff = 0;
+  uint32_t rdlen = objcfg->datasize;
+
+  Reader reader( *objcfg );
+  // open the data object
+  XrdCl::SyncResponseHandler handler1;
+  reader.Open( &handler1 );
+  handler1.WaitForResponse();
+  XrdCl::XRootDStatus *status = handler1.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+
+  // read the data
+  char *rdbuff = new char[rdlen];
+  XrdCl::SyncResponseHandler h;
+  reader.Read( rdoff, rdlen, rdbuff, &h, 0 );
+  h.WaitForResponse();
+  status = h.GetStatus();
+  GTEST_ASSERT_XRDST_NOTOK( *status, XrdCl::errDataError );
+  delete status;
+  delete[] rdbuff;
+
+  // close the data object
+  XrdCl::SyncResponseHandler handler2;
+  reader.Close( &handler2 );
+  handler2.WaitForResponse();
+  status = handler2.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+}
+
+int unlink_cb(const char *fpath, const struct stat *sb, int typeflag, struct FTW *ftwbuf)
+{
+  int rc = remove( fpath );
+  EXPECT_EQ( rc, 0 );
+  return rc;
+}
+
+void XrdEcTests::CleanUp()
+{
+  // delete the data directory
+  nftw( datadir.c_str(), unlink_cb, 64, FTW_DEPTH | FTW_PHYS );
+}
+
+void XrdEcTests::AlignedWriteRaw()
+{
+  char buffer[objcfg->chunksize];
+  StrmWriter writer( *objcfg );
+  // open the data object
+  XrdCl::SyncResponseHandler handler1;
+  writer.Open( &handler1 );
+  handler1.WaitForResponse();
+  XrdCl::XRootDStatus *status = handler1.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+  // write to the data object
+  for( size_t i = 0; i < nbiters; ++i )
+  {
+    memset( buffer, 'A' + i, objcfg->chunksize );
+    writer.Write( objcfg->chunksize, buffer, nullptr );
+    copy_rawdata( buffer, sizeof( buffer ) );
+  }
+  XrdCl::SyncResponseHandler handler2;
+  writer.Close( &handler2 );
+  handler2.WaitForResponse();
+  status = handler2.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+}
+
+void XrdEcTests::VarlenWriteTest( uint32_t wrtlen, bool usecrc32c )
+{
+  // create the data and stripe directories
+  Init( usecrc32c );
+  // open the data object
+  StrmWriter writer( *objcfg );
+  XrdCl::SyncResponseHandler handler1;
+  writer.Open( &handler1 );
+  handler1.WaitForResponse();
+  XrdCl::XRootDStatus *status = handler1.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+  // write the data
+  char     wrtbuff[wrtlen];
+  size_t   bytesleft = nbiters * objcfg->chunksize;
+  size_t   i = 0;
+  while( bytesleft > 0 )
+  {
+    if( wrtlen > bytesleft ) wrtlen = bytesleft;
+    memset( wrtbuff, 'A' + i, wrtlen );
+    writer.Write( wrtlen, wrtbuff, nullptr );
+    copy_rawdata( wrtbuff, wrtlen );
+    bytesleft -= wrtlen;
+    ++i;
+  }
+  XrdCl::SyncResponseHandler handler2;
+  writer.Close( &handler2 );
+  handler2.WaitForResponse();
+  status = handler2.GetStatus();
+  GTEST_ASSERT_XRDST( *status );
+  delete status;
+
+  // verify that we wrote the data correctly
+  Verify();
+  // clean up the data directory
+  CleanUp();
+}
+
diff --git a/tests/XrdEcTests/CMakeLists.txt b/tests/XrdEcTests/CMakeLists.txt
index 7b6ede61b83..e656ab4b585 100644
--- a/tests/XrdEcTests/CMakeLists.txt
+++ b/tests/XrdEcTests/CMakeLists.txt
@@ -1,10 +1,4 @@
 
-include( XRootDCommon )
-include_directories( ${CPPUNIT_INCLUDE_DIRS} ../common )
-
-link_directories( ${ISAL_LIBDIR} )
-include_directories( ${ISAL_INCDIR} )
-
 add_library(
   XrdEcTests MODULE
   MicroTest.cc
@@ -12,7 +6,33 @@ add_library(
 
 target_link_libraries(
   XrdEcTests
-  XrdEc )
+  PRIVATE
+  XrdEc
+  XrdCl
+  XrdUtils
+  ${ISAL_LIBRARIES}
+  ${CPPUNIT_LIBRARIES})
+
+target_include_directories(XrdEcTests PRIVATE ../common ${CPPUNIT_INCLUDE_DIRS} ${ISAL_INCLUDE_DIRS})
+
+foreach(TEST
+  AlignedWriteTest
+  SmallWriteTest
+  BigWriteTest
+  VectorReadTest
+  IllegalVectorReadTest
+  AlignedWrite1MissingTest
+  AlignedWrite2MissingTest
+  AlignedWriteTestIsalCrcNoMt
+  SmallWriteTestIsalCrcNoMt
+  BigWriteTestIsalCrcNoMt
+  AlignedWrite1MissingTestIsalCrcNoMt
+  AlignedWrite2MissingTestIsalCrcNoMt)
+    add_test(NAME XrdEc::${TEST}
+      COMMAND $ $
+        "All Tests/MicroTest/MicroTest::${TEST}"
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+endforeach()
 
 #-------------------------------------------------------------------------------
 # Install
diff --git a/tests/XrdEcTests/MicroTest.cc b/tests/XrdEcTests/MicroTest.cc
index 7668e4c6a66..5b375c6fe4f 100644
--- a/tests/XrdEcTests/MicroTest.cc
+++ b/tests/XrdEcTests/MicroTest.cc
@@ -34,6 +34,8 @@
 
 #include "XrdZip/XrdZipCDFH.hh"
 
+#include "XrdSys/XrdSysPlatform.hh"
+
 #include 
 #include 
 #include 
@@ -280,13 +282,12 @@ void MicroTest::Init( bool usecrc32c )
   objcfg.reset( new ObjCfg( "test.txt", nbdata, nbparity, chsize, usecrc32c, true ) );
   rawdata.clear();
 
-  char cwdbuff[1024];
-  char *cwdptr = getcwd( cwdbuff, sizeof( cwdbuff ) );
-  CPPUNIT_ASSERT( cwdptr );
-  std::string cwd = cwdptr;
+  char tmpdir[MAXPATHLEN];
+  CPPUNIT_ASSERT( getcwd(tmpdir, MAXPATHLEN - 21) );
+  strcat(tmpdir, "/xrootd-xrdec-XXXXXX");
   // create the data directory
-  datadir = cwd + "/data";
-  CPPUNIT_ASSERT( mkdir( datadir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH ) == 0 );
+  CPPUNIT_ASSERT( mkdtemp(tmpdir) );
+  datadir = tmpdir;
   // create a directory for each stripe
   size_t nbstrps = objcfg->nbdata + 2 * objcfg->nbparity;
   for( size_t i = 0; i < nbstrps; ++i )
diff --git a/tests/XrdHttpTests/XrdHttpTests.cc b/tests/XrdHttpTests/XrdHttpTests.cc
index 739b45b7f08..7b7ce778de3 100644
--- a/tests/XrdHttpTests/XrdHttpTests.cc
+++ b/tests/XrdHttpTests/XrdHttpTests.cc
@@ -3,10 +3,11 @@
 #include "XrdHttp/XrdHttpReq.hh"
 #include "XrdHttp/XrdHttpProtocol.hh"
 #include "XrdHttp/XrdHttpChecksumHandler.hh"
+#include "XrdHttp/XrdHttpReadRangeHandler.hh"
 #include 
 #include 
 #include 
-
+#include 
 
 
 using namespace testing;
@@ -164,4 +165,412 @@ TEST(XrdHttpTests, checksumHandlerSelectionTest) {
         handler.configure(configChecksumList);
         ASSERT_EQ(nullptr, handler.getChecksumToRun(reqDigest));
     }
-}
\ No newline at end of file
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerTwoRangesOfSizeEqualToMaxChunkSize) {
+    long long filesize = 8;
+    int rangeBegin = 0;
+    int rangeEnd = 3;
+    int rangeBegin2 = 4;
+    int rangeEnd2 = 7;
+    int readvMaxChunkSize = 4;
+    int readvMaxChunks = 20;
+    int rReqMaxSize = 200;
+    std::stringstream ss;
+    ss << "bytes=" << rangeBegin << "-" << rangeEnd << ", " << rangeBegin2 << "-" << rangeEnd2;
+    std::string rs = ss.str();
+    XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+    XrdHttpReadRangeHandler h(cfg);
+    h.ParseContentRange(rs.c_str());
+    h.SetFilesize(filesize);
+    const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(2, cl.size());
+    ASSERT_EQ(0, cl[0].offset);
+    ASSERT_EQ(4, cl[0].size);
+    ASSERT_EQ(4, cl[1].offset);
+    ASSERT_EQ(4, cl[1].size);
+    ASSERT_EQ(2, ul.size());
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerOneRangeSizeLessThanMaxChunkSize) {
+  long long filesize = 8;
+  int rangeBegin = 0;
+  int rangeEnd = 3;
+  int readvMaxChunkSize = 5;
+  int readvMaxChunks = 20;
+  int rReqMaxSize = 200;
+  std::stringstream ss;
+  ss << "bytes=" << rangeBegin << "-" << rangeEnd;
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+  const XrdHttpIOList &cl = h.NextReadList();
+  ASSERT_EQ(1, cl.size());
+  ASSERT_EQ(0, cl[0].offset);
+  ASSERT_EQ(4, cl[0].size);
+  ASSERT_EQ(1, ul.size());
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerOneRangeSizeGreaterThanMaxChunkSize) {
+  long long filesize = 8;
+  int rangeBegin = 0;
+  int rangeEnd = 7;
+  int readvMaxChunkSize = 3;
+  int readvMaxChunks = 20;
+  int rReqMaxSize = 200;
+  std::stringstream ss;
+  ss << "bytes=" << rangeBegin << "-" << rangeEnd;
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  {
+    const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(1, cl.size());
+    ASSERT_EQ(0, cl[0].offset);
+    ASSERT_EQ(8, cl[0].size);
+    ASSERT_EQ(1, ul.size());
+  }
+  ss.str("");
+  ss << "bytes=0-0," << rangeBegin << "-" << rangeEnd;
+  rs = ss.str();
+  h.reset();
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  {
+    const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(4, cl.size());
+    ASSERT_EQ(0, cl[0].offset);
+    ASSERT_EQ(1, cl[0].size);
+    ASSERT_EQ(0, cl[1].offset);
+    ASSERT_EQ(3, cl[1].size);
+    ASSERT_EQ(3, cl[2].offset);
+    ASSERT_EQ(3, cl[2].size);
+    ASSERT_EQ(6, cl[3].offset);
+    ASSERT_EQ(2, cl[3].size);
+    ASSERT_EQ(2, ul.size());
+  }
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerRange0ToEnd) {
+  long long filesize = 200;
+  int rangeBegin = 0;
+  int readvMaxChunkSize = 4;
+  int readvMaxChunks = 20;
+  int rReqMaxSize = 100;
+  bool start, finish;
+  std::stringstream ss;
+  ss << "bytes=" << rangeBegin << "-" << "\r";
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+  {
+    const XrdHttpIOList &cl1 = h.NextReadList();
+    ASSERT_EQ(1, ul.size());
+    ASSERT_EQ(1, cl1.size());
+    ASSERT_EQ(0, cl1[0].offset);
+    ASSERT_EQ(100, cl1[0].size);
+    ASSERT_EQ(0, h.NotifyReadResult(100, nullptr, start, finish));
+  }
+  {
+    const XrdHttpIOList &cl2 = h.NextReadList();
+    ASSERT_EQ(1, cl2.size());
+    ASSERT_EQ(100, cl2[0].offset);
+    ASSERT_EQ(100, cl2[0].size);
+  }
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerRange5FromEnd) {
+  long long filesize = 200;
+  int rangeEnd = 5;
+  int readvMaxChunkSize = 4;
+  int readvMaxChunks = 20;
+  int rReqMaxSize = 100;
+  bool start, finish;
+  std::stringstream ss;
+  ss << "bytes=-" << rangeEnd << "\r";
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+  {
+    const XrdHttpIOList &cl1 = h.NextReadList();
+    ASSERT_EQ(1, ul.size());
+    ASSERT_EQ(1, cl1.size());
+    ASSERT_EQ(195, cl1[0].offset);
+    ASSERT_EQ(5, cl1[0].size);
+    ASSERT_EQ(0, h.NotifyReadResult(5, nullptr, start, finish));
+  }
+  {
+    const XrdHttpIOList &cl2 = h.NextReadList();
+    ASSERT_EQ(0, cl2.size());
+    const XrdHttpReadRangeHandler::Error &error = h.getError();
+    ASSERT_EQ(false, static_cast(error));
+  }
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerRange0To0) {
+  long long filesize = 8;
+  int rangeBegin = 0;
+  int rangeEnd = 0;
+  int readvMaxChunkSize = 4;
+  int readvMaxChunks = 20;
+  int rReqMaxSize = 100;
+  std::stringstream ss;
+  ss << "bytes=" << rangeBegin << "-" << rangeEnd;
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+  const XrdHttpIOList &cl = h.NextReadList();
+  ASSERT_EQ(1, cl.size());
+  ASSERT_EQ(1, ul.size());
+  ASSERT_EQ(0, ul[0].start);
+  ASSERT_EQ(0, ul[0].end);
+  ASSERT_EQ(0, cl[0].offset);
+  ASSERT_EQ(1, cl[0].size);
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerEndByteGreaterThanFileSize) {
+  long long filesize = 2;
+  int rangeBegin = 0;
+  int rangeEnd = 4;
+  int readvMaxChunkSize = 10;
+  int readvMaxChunks = 20;
+  int rReqMaxSize = 100;
+  std::stringstream ss;
+  ss << "bytes=" << rangeBegin << "-" << rangeEnd;
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+  const XrdHttpIOList &cl = h.NextReadList();
+  ASSERT_EQ(1, cl.size());
+  ASSERT_EQ(0, cl[0].offset);
+  ASSERT_EQ(2, cl[0].size);
+  ASSERT_EQ(1, ul.size());
+  ASSERT_EQ(0, ul[0].start);
+  ASSERT_EQ(1, ul[0].end);
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerRangeBeginGreaterThanFileSize) {
+  long long filesize = 2;
+  int rangeBegin = 4;
+  int rangeEnd = 6;
+  int readvMaxChunkSize = 10;
+  int readvMaxChunks = 20;
+  int rReqMaxSize = 100;
+  std::stringstream ss;
+  ss << "bytes=" << rangeBegin << "-" << rangeEnd;
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+  const XrdHttpIOList &cl = h.NextReadList();
+  ASSERT_EQ(0, ul.size());
+  ASSERT_EQ(0, cl.size());
+  const XrdHttpReadRangeHandler::Error &error = h.getError();
+  ASSERT_EQ(true, static_cast(error));
+  ASSERT_EQ(416, error.httpRetCode);
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerTwoRangesOneOutsideFileExtent) {
+  long long filesize = 20;
+  int rangeBegin1 = 22;
+  int rangeEnd1 = 30;
+  int rangeBegin2 = 4;
+  int rangeEnd2 = 6;
+  int readvMaxChunkSize = 10;
+  int readvMaxChunks = 20;
+  int rReqMaxSize = 100;
+  std::stringstream ss;
+  ss << "bytes=" << rangeBegin1 << "-" << rangeEnd1 << "," << rangeBegin2 << "-" << rangeEnd2;
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+  const XrdHttpIOList &cl = h.NextReadList();
+  ASSERT_EQ(1, ul.size());
+  ASSERT_EQ(4, ul[0].start);
+  ASSERT_EQ(6, ul[0].end);
+  ASSERT_EQ(1, cl.size());
+  ASSERT_EQ(4, cl[0].offset);
+  ASSERT_EQ(3, cl[0].size);
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerMultiChunksSingleRange) {
+  long long filesize = 20;
+  int rangeBegin = 0;
+  int rangeEnd = 15;
+  int readvMaxChunkSize = 3;
+  int readvMaxChunks = 20;
+  int rReqMaxSize = 5;
+  bool start, finish;
+  std::stringstream ss;
+  ss << "bytes=" << rangeBegin << "-" << rangeEnd;
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+  ASSERT_EQ(1, ul.size());
+  ASSERT_EQ(0, ul[0].start);
+  ASSERT_EQ(15, ul[0].end);
+  {
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(1, cl.size());
+    ASSERT_EQ(0, cl[0].offset);
+    ASSERT_EQ(5, cl[0].size);
+    ASSERT_EQ(0, h.NotifyReadResult(5, nullptr, start, finish));
+    ASSERT_EQ(true, start);
+    ASSERT_EQ(false, finish);
+  }
+  {
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(1, cl.size());
+    ASSERT_EQ(5, cl[0].offset);
+    ASSERT_EQ(5, cl[0].size);
+    ASSERT_EQ(0, h.NotifyReadResult(5, nullptr, start, finish));
+    ASSERT_EQ(false, start);
+    ASSERT_EQ(false, finish);
+  }
+  {
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(1, cl.size());
+    ASSERT_EQ(10, cl[0].offset);
+    ASSERT_EQ(5, cl[0].size);
+    ASSERT_EQ(0, h.NotifyReadResult(5, nullptr, start, finish));
+    ASSERT_EQ(false, start);
+    ASSERT_EQ(false, finish);
+  }
+  {
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(1, cl.size());
+    ASSERT_EQ(15, cl[0].offset);
+    ASSERT_EQ(1, cl[0].size);
+    ASSERT_EQ(0, h.NotifyReadResult(1, nullptr, start, finish));
+    ASSERT_EQ(false, start);
+    ASSERT_EQ(true, finish);
+  }
+  {
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(0, cl.size());
+    const XrdHttpReadRangeHandler::Error &error = h.getError();
+    ASSERT_EQ(false, static_cast(error));
+  }
+}
+
+TEST(XrdHttpTests, xrdHttpReadRangeHandlerMultiChunksTwoRanges) {
+  long long filesize = 22;
+  int rangeBegin1 = 0;
+  int rangeEnd1 = 1;
+  int rangeBegin2 = 5;
+  int rangeEnd2 = 21;
+  int readvMaxChunkSize = 3;
+  int readvMaxChunks = 2;
+  int rReqMaxSize = 5;
+  bool start, finish;
+  const XrdHttpReadRangeHandler::UserRange  *ur;
+  std::stringstream ss;
+  ss << "bytes=" << rangeBegin1 << "-" << rangeEnd1 << "," << rangeBegin2 << "-" << rangeEnd2;
+  std::string rs = ss.str();
+  XrdHttpReadRangeHandler::Configuration cfg(readvMaxChunkSize, readvMaxChunks, rReqMaxSize);
+  XrdHttpReadRangeHandler h(cfg);
+  h.ParseContentRange(rs.c_str());
+  h.SetFilesize(filesize);
+  const XrdHttpReadRangeHandler::UserRangeList &ul = h.ListResolvedRanges();
+  ASSERT_EQ(2, ul.size());
+  ASSERT_EQ(0, ul[0].start);
+  ASSERT_EQ(1, ul[0].end);
+  ASSERT_EQ(5, ul[1].start);
+  ASSERT_EQ(21, ul[1].end);
+  {
+    // we get 0-1, 5-7
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(2, cl.size());
+    ASSERT_EQ(0, cl[0].offset);
+    ASSERT_EQ(2, cl[0].size);
+    ASSERT_EQ(5, cl[1].offset);
+    ASSERT_EQ(3, cl[1].size);
+    ASSERT_EQ(0, h.NotifyReadResult(2, &ur, start, finish));
+    ASSERT_EQ(true, start);
+    ASSERT_EQ(false, finish);
+    ASSERT_EQ(0, ur->start);
+    ASSERT_EQ(1, ur->end);
+    ASSERT_EQ(0, h.NotifyReadResult(3, &ur, start, finish));
+    ASSERT_EQ(true, start);
+    ASSERT_EQ(false, finish);
+    ASSERT_EQ(5, ur->start);
+    ASSERT_EQ(21, ur->end);
+  }
+  {
+    // we get 8-12
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(1, cl.size());
+    ASSERT_EQ(8, cl[0].offset);
+    ASSERT_EQ(5, cl[0].size);
+    ASSERT_EQ(0, h.NotifyReadResult(5, &ur, start, finish));
+    ASSERT_EQ(false, start);
+    ASSERT_EQ(false, finish);
+    ASSERT_EQ(5, ur->start);
+    ASSERT_EQ(21, ur->end);
+  }
+  {
+    // we get 13-17
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(1, cl.size());
+    ASSERT_EQ(13, cl[0].offset);
+    ASSERT_EQ(5, cl[0].size);
+    ASSERT_EQ(0, h.NotifyReadResult(5, &ur, start, finish));
+    ASSERT_EQ(false, start);
+    ASSERT_EQ(false, finish);
+    ASSERT_EQ(5, ur->start);
+    ASSERT_EQ(21, ur->end);
+  }
+  {
+    // we get 18-20, 21-21
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(2, cl.size());
+    ASSERT_EQ(18, cl[0].offset);
+    ASSERT_EQ(3, cl[0].size);
+    ASSERT_EQ(21, cl[1].offset);
+    ASSERT_EQ(1, cl[1].size);
+    ASSERT_EQ(0, h.NotifyReadResult(3, &ur, start, finish));
+    ASSERT_EQ(false, start);
+    ASSERT_EQ(false, finish);
+    ASSERT_EQ(5, ur->start);
+    ASSERT_EQ(21, ur->end);
+    ASSERT_EQ(0, h.NotifyReadResult(1, &ur, start, finish));
+    ASSERT_EQ(false, start);
+    ASSERT_EQ(true, finish);
+    ASSERT_EQ(5, ur->start);
+    ASSERT_EQ(21, ur->end);
+  }
+  {
+    const XrdHttpIOList &cl = h.NextReadList();
+    ASSERT_EQ(0, cl.size());
+    const XrdHttpReadRangeHandler::Error &error = h.getError();
+    ASSERT_EQ(false, static_cast(error));
+  }
+}
diff --git a/tests/XrdSsiTests/CMakeLists.txt b/tests/XrdSsiTests/CMakeLists.txt
index 3bbf4d40a1e..a5e019e568f 100644
--- a/tests/XrdSsiTests/CMakeLists.txt
+++ b/tests/XrdSsiTests/CMakeLists.txt
@@ -1,5 +1,6 @@
-
-include( XRootDCommon )
+if ( XRDCL_ONLY )
+  return()
+endif()
 
 add_executable(
   xrdshmap
diff --git a/tests/XrdSsiTests/XrdShMap.cc b/tests/XrdSsiTests/XrdShMap.cc
index a63ef77c17d..709ed8467f8 100644
--- a/tests/XrdSsiTests/XrdShMap.cc
+++ b/tests/XrdSsiTests/XrdShMap.cc
@@ -74,10 +74,10 @@ namespace
 /*                               D e f i n e s                                */
 /******************************************************************************/
   
-#define FMSG(x) xRC|=1,cerr <] [-p ] [-t ]\n\n";
+std::cerr <<"Usage:   xrdshmap [options] [command [command [...]]]\n\n";
+std::cerr <<"options: [-e] [-h {a32|c32|x32}] [-i ] [-p ] [-t ]\n\n";
 
    if (terse) return rc;
 
    if (!uLine) Usage();
-   cerr <= n)
-               {cerr < i) cerr <<'\n' < i) std::cerr <<'\n' <(adler);
-// cerr <<"Z a32 sz=" <(crc);
-// cerr <<"Z c32 sz=" <= 0) std::cout </dev/null; then
+	exit 1
+fi
+
+STATUS=0
+
+HEADERS=$(find "${INCLUDE_DIR}" -type f -name '*.hh')
+PUBLIC_HEADERS=$(grep -E -v '(XrdPosix|private)' <<< "${HEADERS}")
+PRIVATE_HEADERS=$(grep private <<< "${HEADERS}")
+
+# Check public headers without adding private include directory.
+# This ensures public headers do not depend on any private headers.
+
+while IFS=$'\n' read -r HEADER; do
+	"${CXX}" -fsyntax-only ${CXXFLAGS} -I"${INCLUDE_DIR}" "${HEADER}" || STATUS=1
+done <<< "${PUBLIC_HEADERS}"
+
+# Check private headers
+
+while IFS=$'\n' read -r HEADER; do
+	"${CXX}" -fsyntax-only ${CXXFLAGS} -I"${INCLUDE_DIR}"{,/private} "${HEADER}" || STATUS=1
+done <<< "${PRIVATE_HEADERS}"
+
+exit $STATUS
diff --git a/tests/cluster/CMakeLists.txt b/tests/cluster/CMakeLists.txt
new file mode 100644
index 00000000000..2f59c9fb55e
--- /dev/null
+++ b/tests/cluster/CMakeLists.txt
@@ -0,0 +1,39 @@
+if(XRDCL_ONLY)
+  return()
+endif()
+
+execute_process(COMMAND id -u OUTPUT_VARIABLE UID OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+# ensure that we're not root
+if (UID EQUAL 0)
+  return()
+endif()
+
+list(APPEND XRDENV "XRDCP=$")
+list(APPEND XRDENV "XRDFS=$")
+list(APPEND XRDENV "CRC32C=$")
+list(APPEND XRDENV "ADLER32=$")
+list(APPEND XRDENV "XROOTD=$")
+list(APPEND XRDENV "CMSD=$")
+
+foreach(config common metaman man1 man2 srv1 srv2 srv3 srv4)
+  configure_file(${config}.cfg ${config}.cfg @ONLY)
+endforeach()
+
+file(COPY mvdata DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+add_test(NAME XRootD::cluster::start
+  COMMAND sh -c "cp -r ${CMAKE_CURRENT_SOURCE_DIR}/mvdata ${CMAKE_CURRENT_BINARY_DIR} \
+    && ${CMAKE_CURRENT_SOURCE_DIR}/setup.sh start")
+set_tests_properties(XRootD::cluster::start
+  PROPERTIES ENVIRONMENT "${XRDENV}" FIXTURES_SETUP XRootD_Cluster)
+
+add_test(NAME XRootD::cluster::test
+  COMMAND sh -c "${CMAKE_CURRENT_SOURCE_DIR}/test.sh")
+set_tests_properties(XRootD::cluster::test
+  PROPERTIES ENVIRONMENT "${XRDENV}" FIXTURES_REQUIRED XRootD_Cluster)
+
+add_test(NAME XRootD::cluster::stop
+  COMMAND sh -c "${CMAKE_CURRENT_SOURCE_DIR}/setup.sh stop")
+set_tests_properties(XRootD::cluster::stop
+  PROPERTIES ENVIRONMENT "${XRDENV}" FIXTURES_CLEANUP XRootD_Cluster)
diff --git a/tests/cluster/common.cfg b/tests/cluster/common.cfg
new file mode 100644
index 00000000000..47d7ad281de
--- /dev/null
+++ b/tests/cluster/common.cfg
@@ -0,0 +1,25 @@
+all.sitename $name
+
+set pwd = ${PWD}
+set lib = @CMAKE_BINARY_DIR@/src
+
+all.export /
+oss.localroot $pwd/data/$name
+all.adminpath $pwd
+all.pidpath   $pwd
+
+xrd.maxfd strict 8k
+xrd.network cache 5m nodnr norpipa
+
+cms.delay startup 2 lookup 1 qdl 2 suspend 1
+cms.space linger 0 recalc 15 min 2% 1g 5% 2g
+
+ofs.chkpnt enable
+ofs.tpc streams 8 pgm @CMAKE_BINARY_DIR@/src/XrdCl/xrdcp --server
+ofs.ckslib zcrc32 $lib/libXrdCksCalczcrc32.so
+xrootd.chksum zcrc32 chkcgi adler32 crc32c
+
+all.trace    all
+tpc.trace    all
+xrd.trace    all
+xrootd.trace all
diff --git a/tests/cluster/man1.cfg b/tests/cluster/man1.cfg
new file mode 100644
index 00000000000..0ec7042f2ce
--- /dev/null
+++ b/tests/cluster/man1.cfg
@@ -0,0 +1,10 @@
+set name = man1
+
+all.role manager
+all.manager meta localhost:20940
+all.manager localhost:20941
+
+xrd.port 10941 if exec xrootd
+xrd.port 20941 if exec cmsd
+
+continue @CMAKE_CURRENT_BINARY_DIR@/common.cfg
diff --git a/tests/cluster/man2.cfg b/tests/cluster/man2.cfg
new file mode 100644
index 00000000000..077129bdfde
--- /dev/null
+++ b/tests/cluster/man2.cfg
@@ -0,0 +1,10 @@
+set name = man2
+
+all.role manager
+all.manager meta localhost:20940
+all.manager localhost:20942
+
+xrd.port 10942 if exec xrootd
+xrd.port 20942 if exec cmsd
+
+continue @CMAKE_CURRENT_BINARY_DIR@/common.cfg
diff --git a/tests/cluster/metaman.cfg b/tests/cluster/metaman.cfg
new file mode 100644
index 00000000000..842d20bc8f2
--- /dev/null
+++ b/tests/cluster/metaman.cfg
@@ -0,0 +1,9 @@
+set name = metaman
+
+all.role meta manager
+all.manager meta localhost:20940
+
+xrd.port 10940 if exec xrootd
+xrd.port 20940 if exec cmsd
+
+continue @CMAKE_CURRENT_BINARY_DIR@/common.cfg
diff --git a/tests/cluster/mvdata/data.zip b/tests/cluster/mvdata/data.zip
new file mode 100644
index 00000000000..b5d2fb904f1
Binary files /dev/null and b/tests/cluster/mvdata/data.zip differ
diff --git a/tests/cluster/mvdata/input1.meta4 b/tests/cluster/mvdata/input1.meta4
new file mode 100644
index 00000000000..1852892387d
--- /dev/null
+++ b/tests/cluster/mvdata/input1.meta4
@@ -0,0 +1,8 @@
+ 
+ 
+   
+     A_file:output.dat
+     root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/input1.metalink b/tests/cluster/mvdata/input1.metalink
new file mode 100644
index 00000000000..cd80ca4a127
--- /dev/null
+++ b/tests/cluster/mvdata/input1.metalink
@@ -0,0 +1,11 @@
+ 
+ 
+   
+     
+       
+         root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+       
+     
+   
+ 
+
diff --git a/tests/cluster/mvdata/input2.meta4 b/tests/cluster/mvdata/input2.meta4
new file mode 100644
index 00000000000..1bacadecef5
--- /dev/null
+++ b/tests/cluster/mvdata/input2.meta4
@@ -0,0 +1,9 @@
+ 
+ 
+   
+     A_file:output.dat
+     root://localhost:10944//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+     root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/input2.metalink b/tests/cluster/mvdata/input2.metalink
new file mode 100644
index 00000000000..d4e2dae0cf0
--- /dev/null
+++ b/tests/cluster/mvdata/input2.metalink
@@ -0,0 +1,12 @@
+ 
+ 
+   
+     
+       
+         root://localhost:10944//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+         root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+       
+     
+   
+ 
+
diff --git a/tests/cluster/mvdata/input3.meta4 b/tests/cluster/mvdata/input3.meta4
new file mode 100644
index 00000000000..639489f3672
--- /dev/null
+++ b/tests/cluster/mvdata/input3.meta4
@@ -0,0 +1,9 @@
+ 
+ 
+   
+     A_file:output.dat
+     40e5fdb0
+     root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/input3.metalink b/tests/cluster/mvdata/input3.metalink
new file mode 100644
index 00000000000..8067d41a24d
--- /dev/null
+++ b/tests/cluster/mvdata/input3.metalink
@@ -0,0 +1,14 @@
+ 
+ 
+   
+     
+       
+         40e5fdb0
+       
+       
+         root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+       
+     
+   
+ 
+
diff --git a/tests/cluster/mvdata/input4.meta4 b/tests/cluster/mvdata/input4.meta4
new file mode 100644
index 00000000000..68e068a520e
--- /dev/null
+++ b/tests/cluster/mvdata/input4.meta4
@@ -0,0 +1,10 @@
+ 
+ 
+   
+     A_file:output.dat
+     40e5fdb0
+     root://localhost:10944//data/1db882c8-8cd6-4df1-941f-ce669bad3458.dat
+     root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/input4.metalink b/tests/cluster/mvdata/input4.metalink
new file mode 100644
index 00000000000..e95cfbd1531
--- /dev/null
+++ b/tests/cluster/mvdata/input4.metalink
@@ -0,0 +1,15 @@
+ 
+ 
+   
+     
+       
+         40e5fdb0
+       
+       
+         root://localhost:10944//data/1db882c8-8cd6-4df1-941f-ce669bad3458.dat
+         root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+       
+     
+   
+ 
+
diff --git a/tests/cluster/mvdata/input5.meta4 b/tests/cluster/mvdata/input5.meta4
new file mode 100644
index 00000000000..30438fd9521
--- /dev/null
+++ b/tests/cluster/mvdata/input5.meta4
@@ -0,0 +1,11 @@
+ 
+ 
+   
+     A_file:output.dat
+     f024c8e5
+     851fef8c18d878b31366d1aabeefdead
+     40e5fdb0
+     root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/input5.metalink b/tests/cluster/mvdata/input5.metalink
new file mode 100644
index 00000000000..e745d2b64fe
--- /dev/null
+++ b/tests/cluster/mvdata/input5.metalink
@@ -0,0 +1,16 @@
+ 
+ 
+   
+     
+      
+         f024c8e5
+         851fef8c18d878b31366d1aabeefdead
+         40e5fdb0
+        
+       
+         root://localhost:10943//data/b74d025e-06d6-43e8-91e1-a862feb03c84.dat
+       
+     
+   
+ 
+
diff --git a/tests/cluster/mvdata/large.zip b/tests/cluster/mvdata/large.zip
new file mode 100644
index 00000000000..c0612aa89c0
Binary files /dev/null and b/tests/cluster/mvdata/large.zip differ
diff --git a/tests/cluster/mvdata/mlFileTest1.meta4 b/tests/cluster/mvdata/mlFileTest1.meta4
new file mode 100644
index 00000000000..3647cf20e4b
--- /dev/null
+++ b/tests/cluster/mvdata/mlFileTest1.meta4
@@ -0,0 +1,7 @@
+ 
+ 
+   
+     root://localhost:10943//data/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/mlFileTest2.meta4 b/tests/cluster/mvdata/mlFileTest2.meta4
new file mode 100644
index 00000000000..a78aaddc3b6
--- /dev/null
+++ b/tests/cluster/mvdata/mlFileTest2.meta4
@@ -0,0 +1,8 @@
+ 
+ 
+   
+     root://localhost:10943//data/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat
+     root://localhost:10944//data/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/mlFileTest3.meta4 b/tests/cluster/mvdata/mlFileTest3.meta4
new file mode 100644
index 00000000000..fd810e15faa
--- /dev/null
+++ b/tests/cluster/mvdata/mlFileTest3.meta4
@@ -0,0 +1,8 @@
+ 
+ 
+   
+     root://localhost:10945//data/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat
+     root://localhost:10944//data/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/mlFileTest4.meta4 b/tests/cluster/mvdata/mlFileTest4.meta4
new file mode 100644
index 00000000000..194665b5936
--- /dev/null
+++ b/tests/cluster/mvdata/mlFileTest4.meta4
@@ -0,0 +1,8 @@
+ 
+ 
+   
+     root://localhost:10945//data/3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat
+     root://localhost:10944//data/3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/mlTpcTest.meta4 b/tests/cluster/mvdata/mlTpcTest.meta4
new file mode 100644
index 00000000000..31b5287d8a4
--- /dev/null
+++ b/tests/cluster/mvdata/mlTpcTest.meta4
@@ -0,0 +1,9 @@
+ 
+ 
+   
+     b43f105c
+     16777216
+     root://localhost:10943//data/cb4aacf1-6f28-42f2-b68a-90a73460f424.dat
+   
+ 
+
diff --git a/tests/cluster/mvdata/mlZipTest.meta4 b/tests/cluster/mvdata/mlZipTest.meta4
new file mode 100644
index 00000000000..661627c54bd
--- /dev/null
+++ b/tests/cluster/mvdata/mlZipTest.meta4
@@ -0,0 +1,9 @@
+ 
+ 
+   
+     75a16a5b
+     4047392
+     root://localhost:10946//data/large.zip
+   
+ 
+
diff --git a/tests/cluster/setup.sh b/tests/cluster/setup.sh
new file mode 100755
index 00000000000..3c5624359f9
--- /dev/null
+++ b/tests/cluster/setup.sh
@@ -0,0 +1,197 @@
+#!/usr/bin/env bash
+
+######
+# Starts a cluster configuration locally, instead of using
+# containers.
+#####
+
+set -e
+
+: ${XROOTD:=$(command -v xrootd)}
+: ${CMSD:=$(command -v cmsd)}
+: ${OPENSSL:=$(command -v openssl)}
+: ${CRC32C:=$(command -v xrdcrc32c)}
+: ${STAT:=$(command -v stat)}
+
+servernames=("metaman" "man1" "man2" "srv1" "srv2" "srv3" "srv4")
+datanodes=("srv1" "srv2" "srv3" "srv4")
+
+DATAFOLDER="./data"
+TMPDATAFOLDER="./rout"
+PREDEF="./mvdata"
+
+filenames=("1db882c8-8cd6-4df1-941f-ce669bad3458.dat"
+       "3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat"
+       "7235b5d1-cede-4700-a8f9-596506b4cc38.dat"
+       "7e480547-fe1a-4eaf-a210-0f3927751a43.dat"
+       "89120cec-5244-444c-9313-703e4bee72de.dat"
+       "a048e67f-4397-4bb8-85eb-8d7e40d90763.dat"
+       "b3d40b3f-1d15-4ad3-8cb5-a7516acb2bab.dat"
+       "b74d025e-06d6-43e8-91e1-a862feb03c84.dat"
+       "cb4aacf1-6f28-42f2-b68a-90a73460f424.dat"
+       "cef4d954-936f-4945-ae49-60ec715b986e.dat")
+
+filesize() {
+	case $(uname) in
+	Darwin) ${STAT} -f"%z" $1 ;;
+	Linux)  ${STAT} -c"%s" $1 ;;
+	*)      ${STAT} -c"%s" $1 ;;
+	esac
+}
+
+formatfiles() {
+	case $(uname) in
+              Darwin)
+               sed -i '' "s|.*|$new_size|" $1
+               sed -i '' "s|.*|$new_hash|" $1
+               sed -i '' "s|.*|$new_hash|" $1
+               ;;
+              Linux)
+               sed -i "s|.*|$new_size|" $1
+               sed -i "s|.*|$new_hash|" $1
+               sed -i "s|.*|$new_hash|" $1
+               ;;
+              *)
+               sed -i "s|.*|$new_size|" $1
+               sed -i "s|.*|$new_hash|" $1
+               sed -i "s|.*|$new_hash|" $1
+               ;;
+	esac
+}
+
+generate(){
+
+       # check if files are in the data directory already...
+       if [[ -e ${DATAFOLDER}/${i} ]]; then
+              return
+       fi
+
+       mkdir -p ${TMPDATAFOLDER}
+
+       for i in ${filenames[@]}; do
+              ${OPENSSL} rand -out "${TMPDATAFOLDER}/${i}" $(( 2**24 ))
+       done
+
+       # correct the info inside of metalink files
+       insertFileInfo
+
+       # create local srv directories
+       echo "Creating directories for each instance..."
+
+       for i in ${servernames[@]}; do
+              mkdir -p ${DATAFOLDER}/${i}/data
+       done
+
+       for i in ${datanodes[@]}; do
+              mkdir -p ${DATAFOLDER}/${i}/data/bigdir
+              cd ${DATAFOLDER}/${i}/data/bigdir
+              for i in `seq 1000`;
+                     do touch `uuidgen`.dat;
+              done
+              cd - >/dev/null
+       done
+
+       for i in ${servernames[@]}; do
+              if [[ ${i} == 'metaman' ]] ; then
+                     # download the a test file for upload tests
+                     mkdir -p ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat ${DATAFOLDER}/${i}/data/testFile.dat
+              fi
+
+              # download the test files for 'srv1'
+              if [[ ${i} == 'srv1' ]] ; then
+                     cp ${TMPDATAFOLDER}/a048e67f-4397-4bb8-85eb-8d7e40d90763.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/b3d40b3f-1d15-4ad3-8cb5-a7516acb2bab.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/b74d025e-06d6-43e8-91e1-a862feb03c84.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/cb4aacf1-6f28-42f2-b68a-90a73460f424.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/cef4d954-936f-4945-ae49-60ec715b986e.dat ${DATAFOLDER}/${i}/data
+                     mkdir -p ${DATAFOLDER}/${i}/data/metalink
+                     cp ${PREDEF}/input*.meta* ${DATAFOLDER}/${i}/data/metalink/
+                     cp ${PREDEF}/ml*.meta*    ${DATAFOLDER}/${i}/data/metalink/
+              fi
+
+              # download the test files for 'srv2' and add another instance on 1099
+              if [[ ${i} == 'srv2' ]] ; then
+                     cp ${TMPDATAFOLDER}/1db882c8-8cd6-4df1-941f-ce669bad3458.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/7235b5d1-cede-4700-a8f9-596506b4cc38.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/7e480547-fe1a-4eaf-a210-0f3927751a43.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/89120cec-5244-444c-9313-703e4bee72de.dat ${DATAFOLDER}/${i}/data
+              fi
+
+              # download the test files for 'srv3'
+              if [[ ${i} == 'srv3' ]] ; then
+                     cp ${TMPDATAFOLDER}/1db882c8-8cd6-4df1-941f-ce669bad3458.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/3c9a9dd8-bc75-422c-b12c-f00604486cc1.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/89120cec-5244-444c-9313-703e4bee72de.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/b74d025e-06d6-43e8-91e1-a862feb03c84.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/cef4d954-936f-4945-ae49-60ec715b986e.dat ${DATAFOLDER}/${i}/data
+              fi
+
+              # download the test files for 'srv4'
+              if [[ ${i} == 'srv4' ]] ; then
+                     cp ${TMPDATAFOLDER}/1db882c8-8cd6-4df1-941f-ce669bad3458.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/7e480547-fe1a-4eaf-a210-0f3927751a43.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/89120cec-5244-444c-9313-703e4bee72de.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/b74d025e-06d6-43e8-91e1-a862feb03c84.dat ${DATAFOLDER}/${i}/data
+                     cp ${TMPDATAFOLDER}/cef4d954-936f-4945-ae49-60ec715b986e.dat ${DATAFOLDER}/${i}/data
+                     cp ${PREDEF}/data.zip                                 ${DATAFOLDER}/${i}/data
+                     cp ${PREDEF}/large.zip                                ${DATAFOLDER}/${i}/data
+              fi
+       done
+
+       rm -rf ${TMPDATAFOLDER}
+}
+
+start(){
+       generate
+       set -x
+       # start for each component
+       for i in "${servernames[@]}"; do
+              ${XROOTD} -b -k fifo -n ${i} -l xrootd.log -s xrootd.pid -c ${i}.cfg
+       done
+
+       # start cmsd in the redirectors
+       for i in "${servernames[@]}"; do
+              ${CMSD} -b -k fifo -n ${i} -l cmsd.log -s cmsd.pid -c ${i}.cfg
+       done
+}
+
+stop() {
+	for i in "${servernames[@]}"; do
+		if [[ -d "${i}" ]]; then
+			kill -s TERM $(cat ${i}/cmsd.pid)
+			kill -s TERM $(cat ${i}/xrootd.pid)
+			rm -rf "${i}"
+		fi
+	done
+}
+
+insertFileInfo() {
+       # modifies metalink data
+       for file in ${filenames[@]}; do
+              for i in ${PREDEF}/*.meta*; do
+                     # update size and hash
+                     if grep -q $file $i; then
+                            echo "Pattern ${file} found in ${i}!!"
+                            new_size=$(filesize ${TMPDATAFOLDER}/${file})
+                            new_hash=$(${CRC32C} < ${TMPDATAFOLDER}/${file} | cut -d' '  -f1)
+                            $(formatfiles $i)
+                     else
+                            echo "URL not found in the XML."
+                     fi
+              done
+       done
+}
+
+usage() {
+       echo $0 start or stop
+}
+
+[[ $# == 0 ]] && usage && exit 0
+
+CMD=$1
+shift
+[[ $(type -t ${CMD}) == "function" ]] || die "unknown command: ${CMD}"
+$CMD $@
+
diff --git a/tests/cluster/srv1.cfg b/tests/cluster/srv1.cfg
new file mode 100644
index 00000000000..b99d8634336
--- /dev/null
+++ b/tests/cluster/srv1.cfg
@@ -0,0 +1,7 @@
+set name = srv1
+
+all.role server
+all.manager localhost:20941
+xrd.port 10943
+
+continue @CMAKE_CURRENT_BINARY_DIR@/common.cfg
diff --git a/tests/cluster/srv2.cfg b/tests/cluster/srv2.cfg
new file mode 100644
index 00000000000..1f07efdeaf6
--- /dev/null
+++ b/tests/cluster/srv2.cfg
@@ -0,0 +1,7 @@
+set name = srv2
+
+all.role server
+all.manager localhost:20941
+xrd.port 10944
+
+continue @CMAKE_CURRENT_BINARY_DIR@/common.cfg
diff --git a/tests/cluster/srv3.cfg b/tests/cluster/srv3.cfg
new file mode 100644
index 00000000000..b5eae6bfc0f
--- /dev/null
+++ b/tests/cluster/srv3.cfg
@@ -0,0 +1,7 @@
+set name = srv3
+
+all.role server
+all.manager localhost:20942
+xrd.port 10945
+
+continue @CMAKE_CURRENT_BINARY_DIR@/common.cfg
diff --git a/tests/cluster/srv4.cfg b/tests/cluster/srv4.cfg
new file mode 100644
index 00000000000..ae7f6bc38f6
--- /dev/null
+++ b/tests/cluster/srv4.cfg
@@ -0,0 +1,7 @@
+set name = srv4
+
+all.role server
+all.manager localhost:20942
+xrd.port 10946
+
+continue @CMAKE_CURRENT_BINARY_DIR@/common.cfg
diff --git a/tests/cluster/test.sh b/tests/cluster/test.sh
new file mode 100755
index 00000000000..47e2389de7a
--- /dev/null
+++ b/tests/cluster/test.sh
@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+
+# macOS, as of now, cannot run this test because of the 'declare -A'
+# command that we use later, so we just skip this test (sorry apple users)
+if [[ $(uname) == "Darwin" ]]; then
+       exit 0
+fi
+
+# we probably need all of these still
+: ${ADLER32:=$(command -v xrdadler32)}
+: ${CRC32C:=$(command -v xrdcrc32c)}
+: ${XRDCP:=$(command -v xrdcp)}
+: ${XRDFS:=$(command -v xrdfs)}
+: ${OPENSSL:=$(command -v openssl)}
+: ${HOST_METAMAN:=root://localhost:10940}
+: ${HOST_MAN1:=root://localhost:10941}
+: ${HOST_MAN2:=root://localhost:10942}
+: ${HOST_SRV1:=root://localhost:10943}
+: ${HOST_SRV2:=root://localhost:10944}
+: ${HOST_SRV3:=root://localhost:10945}
+: ${HOST_SRV4:=root://localhost:10946}
+
+# checking for command presence
+for PROG in ${ADLER32} ${CRC32C} ${XRDCP} ${XRDFS} ${OPENSSL}; do
+       if [[ ! -x "${PROG}" ]]; then
+               echo 1>&2 "$(basename $0): error: '${PROG}': command not found"
+               exit 1
+       fi
+done
+
+# This script assumes that ${host} exports an empty / as read/write.
+# It also assumes that any authentication required is already setup.
+
+set -xe
+
+echo "xrdcp/fs-test1"
+${XRDCP} --version
+
+for host in "${!hosts[@]}"; do
+       ${XRDFS} ${hosts[$host]} query config version
+done
+
+# query some common server configurations
+
+CONFIG_PARAMS=( version role sitename )
+
+for PARAM in ${CONFIG_PARAMS[@]}; do
+       for host in "${!hosts[@]}"; do
+              ${XRDFS} ${hosts[$host]} query config ${PARAM}
+       done
+done
+
+# some extra query commands that don't make any changes
+${XRDFS} ${HOST_METAMAN} stat /
+${XRDFS} ${HOST_METAMAN} statvfs /
+${XRDFS} ${HOST_METAMAN} spaceinfo /
+
+RMTDATADIR="/srvdata"
+LCLDATADIR="${PWD}/localdata"  # client folder
+
+mkdir -p ${LCLDATADIR}
+
+# hostname-address pair, so that we can keep track of files more easily
+declare -A hosts
+hosts["metaman"]="${HOST_METAMAN}"
+hosts["man1"]="${HOST_MAN1}"
+hosts["man2"]="${HOST_MAN2}"
+hosts["srv1"]="${HOST_SRV1}"
+hosts["srv2"]="${HOST_SRV2}"
+hosts["srv3"]="${HOST_SRV3}"
+hosts["srv4"]="${HOST_SRV4}"
+
+cleanup() {
+       echo "Error occured. Cleaning up..."
+       for host in "${!hosts[@]}"; do
+              rm -rf ${LCLDATADIR}/${host}.dat
+              rm -rf ${LCLDATADIR}/${host}.ref
+       done
+}
+trap "cleanup; exit 1" ABRT
+
+# create local files with random contents using OpenSSL
+echo "Creating files for each instance..."
+
+for host in "${!hosts[@]}"; do
+       ${OPENSSL} rand -out "${LCLDATADIR}/${host}.ref" $((1024 * $RANDOM))
+done
+
+# upload local files to the servers in parallel
+echo "Uploading files..."
+
+
+for host in "${!hosts[@]}"; do
+       ${XRDCP} ${LCLDATADIR}/${host}.ref ${hosts[$host]}/${RMTDATADIR}/${host}.ref
+done
+
+# list uploaded files, then download them to check for corruption
+echo "Downloading them back..."
+
+for host in "${!hosts[@]}"; do
+       ${XRDFS} ${hosts[$host]} ls -l ${RMTDATADIR}
+done
+
+echo "Downloading them back... pt2"
+
+for host in "${!hosts[@]}"; do
+       ${XRDCP} ${hosts[$host]}/${RMTDATADIR}/${host}.ref ${LCLDATADIR}/${host}.dat
+done
+
+# check that all checksums for downloaded files match
+echo "Comparing checksum..."
+
+for host in "${!hosts[@]}"; do
+       REF32C=$(${CRC32C} < ${LCLDATADIR}/${host}.ref | cut -d' '  -f1)
+       NEW32C=$(${CRC32C} < ${LCLDATADIR}/${host}.dat | cut -d' '  -f1)
+       SRV32C=$(${XRDFS} ${hosts[$host]} query checksum ${RMTDATADIR}/${host}.ref?cks.type=crc32c | cut -d' ' -f2)
+
+       REFA32=$(${ADLER32} < ${LCLDATADIR}/${host}.ref | cut -d' '  -f1)
+       NEWA32=$(${ADLER32} < ${LCLDATADIR}/${host}.dat | cut -d' '  -f1)
+       SRVA32=$(${XRDFS} ${hosts[$host]} query checksum ${RMTDATADIR}/${host}.ref?cks.type=adler32 | cut -d' ' -f2)
+       echo "${host}:  crc32c: reference: ${REF32C}, server: ${SRV32C}, downloaded: ${REF32C}"
+       echo "${host}: adler32: reference: ${NEWA32}, server: ${SRVA32}, downloaded: ${NEWA32}"
+
+       if [[ "${NEWA32}" != "${REFA32}" || "${SRVA32}" != "${REFA32}" ]]; then
+               echo 1>&2 "$(basename $0): error: adler32 checksum check failed for file: ${host}.dat"
+               exit 1r
+       fi
+       if [[ "${NEW32C}" != "${REF32C}" || "${SRV32C}" != "${REF32C}" ]]; then
+               echo 1>&2 "$(basename $0): error: crc32 checksum check failed for file: ${host}.dat"
+               exit 1
+       fi
+done
+
+echo "All good! Now removing stuff..."
+
+for host in "${!hosts[@]}"; do
+       ${XRDFS} ${HOST_METAMAN} rm ${RMTDATADIR}/${host}.ref &
+       rm ${LCLDATADIR}/${host}.dat &
+done
+
+wait
+
+${XRDFS} ${HOST_METAMAN} rmdir ${RMTDATADIR}
+
+echo "ALL TESTS PASSED"
+exit 0
diff --git a/tests/common/CMakeLists.txt b/tests/common/CMakeLists.txt
index c40c234e990..5497a7116dc 100644
--- a/tests/common/CMakeLists.txt
+++ b/tests/common/CMakeLists.txt
@@ -1,5 +1,4 @@
 
-include(XRootDCommon)
 
 add_library(
   XrdClTestsHelper SHARED
@@ -16,6 +15,9 @@ target_link_libraries(
   XrdCl
   XrdUtils)
 
+target_include_directories(
+  XrdClTestsHelper PUBLIC ${CPPUNIT_INCLUDE_DIRS})
+
 add_executable(
   test-runner
   TextRunner.cc
@@ -27,6 +29,9 @@ target_link_libraries(
   ${CPPUNIT_LIBRARIES}
   ${CMAKE_THREAD_LIBS_INIT})
 
+target_include_directories(
+  test-runner PRIVATE ${CPPUNIT_INCLUDE_DIRS})
+
 #-------------------------------------------------------------------------------
 # Install
 #-------------------------------------------------------------------------------
diff --git a/tests/common/CppUnitXrdHelpers.hh b/tests/common/CppUnitXrdHelpers.hh
index 99925bb2726..b4634433133 100644
--- a/tests/common/CppUnitXrdHelpers.hh
+++ b/tests/common/CppUnitXrdHelpers.hh
@@ -25,18 +25,18 @@
 
 #define CPPUNIT_ASSERT_XRDST_NOTOK( x, err )         \
 {                                                    \
-  XrdCl::XRootDStatus st = x;                        \
+  XrdCl::XRootDStatus _st = x;                       \
   std::string msg = "["; msg += #x; msg += "]: ";    \
-  msg += st.ToStr();                                 \
-  CPPUNIT_ASSERT_MESSAGE( msg, !st.IsOK() && st.code == err ); \
+  msg += _st.ToStr();                                \
+  CPPUNIT_ASSERT_MESSAGE( msg, !_st.IsOK() && _st.code == err ); \
 }
 
 #define CPPUNIT_ASSERT_XRDST( x )                    \
 {                                                    \
-  XrdCl::XRootDStatus st = x;                        \
+  XrdCl::XRootDStatus _st = x;                       \
   std::string msg = "["; msg += #x; msg += "]: ";    \
-  msg += st.ToStr();                                 \
-  CPPUNIT_ASSERT_MESSAGE( msg, st.IsOK() );          \
+  msg += _st.ToStr();                                \
+  CPPUNIT_ASSERT_MESSAGE( msg, _st.IsOK() );         \
 }
 
 #define CPPUNIT_ASSERT_ERRNO( x )                    \
diff --git a/tests/common/Server.cc b/tests/common/Server.cc
index 6097fd09770..6f9e7cd7304 100644
--- a/tests/common/Server.cc
+++ b/tests/common/Server.cc
@@ -26,6 +26,7 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 
diff --git a/tests/common/TestEnv.cc b/tests/common/TestEnv.cc
index ed526347265..ec6fa4f5726 100644
--- a/tests/common/TestEnv.cc
+++ b/tests/common/TestEnv.cc
@@ -30,19 +30,27 @@ XrdCl::Log *TestEnv::sLog       = 0;
 //------------------------------------------------------------------------------
 TestEnv::TestEnv()
 {
-  PutString( "MainServerURL",    "localhost:1094" );
-  PutString( "Manager1URL",      "man1:1094" );
-  PutString( "Manager2URL",      "man2:1094" );
-  PutString( "DiskServerURL",    "localhost:1094" );
+  PutString( "MainServerURL",    "localhost:10940" );
+  PutString( "Manager1URL",      "localhost:10941" );
+  PutString( "Manager2URL",      "localhost:10942" );
+  PutString( "Server1URL",       "localhost:10943" );
+  PutString( "Server2URL",       "localhost:10944" );
+  PutString( "Server3URL",       "localhost:10945" );
+  PutString( "Server4URL",       "localhost:10946" );
+  PutString( "DiskServerURL",    "localhost:10940" );
   PutString( "DataPath",         "/data"         );
   PutString( "RemoteFile",       "/data/cb4aacf1-6f28-42f2-b68a-90a73460f424.dat" );
   PutString( "LocalFile",        "/data/testFile.dat" );
   PutString( "MultiIPServerURL", "multiip:1099" );
-
+  PutString( "LocalDataPath",    "../cluster/data" );
   ImportString( "MainServerURL",    "XRDTEST_MAINSERVERURL" );
   ImportString( "DiskServerURL",    "XRDTEST_DISKSERVERURL" );
   ImportString( "Manager1URL",      "XRDTEST_MANAGER1URL" );
   ImportString( "Manager2URL",      "XRDTEST_MANAGER2URL" );
+  ImportString( "Server1URL",       "XRDTEST_SERVER1URL" );
+  ImportString( "Server2URL",       "XRDTEST_SERVER2URL" );
+  ImportString( "Server3URL",       "XRDTEST_SERVER3URL" );
+  ImportString( "Server4URL",       "XRDTEST_SERVER4URL" );
   ImportString( "DataPath",         "XRDTEST_DATAPATH" );
   ImportString( "LocalFile",        "XRDTEST_LOCALFILE" );
   ImportString( "RemoteFile",       "XRDTEST_REMOTEFILE" );
diff --git a/tests/common/TextRunner.cc b/tests/common/TextRunner.cc
index 77d8f92d8b7..26f5ff86514 100644
--- a/tests/common/TextRunner.cc
+++ b/tests/common/TextRunner.cc
@@ -149,6 +149,5 @@ int main( int argc, char **argv)
     new CppUnit::CompilerOutputter( &runner.result(), std::cerr ) );
 
   bool wasSuccessful = runner.run();
-  dlclose( libHandle );
   return wasSuccessful ? 0 : 1;
 }
diff --git a/tests/post-install.sh b/tests/post-install.sh
new file mode 100755
index 00000000000..1215bcf79d0
--- /dev/null
+++ b/tests/post-install.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# This script is meant as a basic test to be run post-installation
+# to catch problems such as bad RPATHs, Python bindings not able to
+# find XrdCl libraries post-installation, etc. Setting variables as
+# LD_LIBRARY_PATH, PYTHONPATH, etc, may "fix" a problem caught by
+# this script, but most likely the real fix would be to set correct
+# relative RPATHs such that everything works without any extra steps
+# on the part of the user. PYTHONPATH may be exceptionally set when
+# the Python bindings are intentionally installed into a custom path.
+
+set -e
+
+: "${XRDCP:=$(command -v xrdcp)}"
+: "${XRDFS:=$(command -v xrdfs)}"
+: "${PYTHON:=$(command -v python3 || command -v python)}"
+
+for PROG in ${XRDCP} ${XRDFS} ${PYTHON}; do
+	if [[ ! -x ${PROG} ]]; then
+		echo 1>&2 "$(basename "$0"): error: '${PROG}': command not found"
+		exit 1
+	fi
+done
+
+V=$(xrdcp --version 2>&1)
+echo "Using ${XRDCP} (${V#v})"
+echo "Using ${PYTHON} ($(${PYTHON} --version))"
+${PYTHON} -m pip show xrootd
+${PYTHON} -c 'import XRootD; print(XRootD)'
+${PYTHON} -c 'import pyxrootd; print(pyxrootd)'
+${PYTHON} -c 'from XRootD import client; print(client)'
+${PYTHON} -c 'from XRootD import client; print(client.FileSystem("root://localhost"))'
diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt
new file mode 100644
index 00000000000..e18493cd931
--- /dev/null
+++ b/utils/CMakeLists.txt
@@ -0,0 +1,13 @@
+if( NOT XRDCL_LIB_ONLY )
+  configure_file(xrootd-config xrootd-config @ONLY)
+  install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/xrootd-config DESTINATION ${CMAKE_INSTALL_BINDIR})
+endif()
+
+if( NOT XRDCL_ONLY )
+  install(PROGRAMS
+    ${CMAKE_SOURCE_DIR}/utils/XrdCmsNotify.pm
+    ${CMAKE_SOURCE_DIR}/utils/netchk
+    ${CMAKE_SOURCE_DIR}/utils/XrdOlbMonPerf
+    ${CMAKE_SOURCE_DIR}/utils/cms_monPerf
+    DESTINATION ${CMAKE_INSTALL_DATADIR}/xrootd/utils)
+endif()
diff --git a/utils/xrootd-config b/utils/xrootd-config
index 685fdc8d3ce..2f00e163191 100755
--- a/utils/xrootd-config
+++ b/utils/xrootd-config
@@ -23,10 +23,10 @@
 # or submit itself to any jurisdiction.
 #-------------------------------------------------------------------------------
 
-version=__VERSION__
-prefix=__PREFIX__
-includedir=${prefix}/__INCLUDEDIR__/xrootd
-plugin_version=__PLUGIN_VERSION__
+version=@XRootD_VERSION_STRING@
+prefix=@CMAKE_INSTALL_PREFIX@
+includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@/xrootd
+plugin_version=@PLUGIN_VERSION@
 
 usage()
 {
@@ -58,7 +58,7 @@ while test $# -gt 0; do
       echo $prefix
       ;;
     --version)
-      echo $version
+      echo ${version#v}
       ;;
     --cflags)
       if test "$includedir" != "/usr/include" ; then
diff --git a/xrootd.spec b/xrootd.spec
new file mode 100644
index 00000000000..415e53a1956
--- /dev/null
+++ b/xrootd.spec
@@ -0,0 +1,1063 @@
+%bcond_with    asan
+%bcond_with    ceph
+%bcond_with    clang
+%bcond_with    compat
+%bcond_with    docs
+%bcond_with    git
+
+%bcond_without tests
+%bcond_without xrdec
+
+%if %{?rhel}%{!?rhel:0} == 7
+%bcond_with    openssl11
+%endif
+
+%global compat_version 4.12.9
+%global devtoolset devtoolset-7
+%global llvmtoolset llvm-toolset-7
+
+Name:		xrootd
+Epoch:		1
+Release:	1%{?dist}%{?with_clang:.clang}%{?with_asan:.asan}%{?with_openssl11:.ssl11}
+Summary:	Extended ROOT File Server
+Group:		System Environment/Daemons
+License:	LGPL-3.0-or-later AND BSD-2-Clause AND BSD-3-Clause AND curl AND MIT AND Zlib
+URL:		https://xrootd.slac.stanford.edu
+
+%if !%{with git}
+Version:	5.6.9
+Source0:	%{url}/download/v%{version}/%{name}-%{version}.tar.gz
+%else
+%define git_version %(tar xzf %{_sourcedir}/%{name}.tar.gz -O xrootd/VERSION)
+%define src_version %(sed -e "s/%%(describe)/v5.7-rc%(date +%%Y%%m%%d)/" <<< "%git_version")
+%define rpm_version %(sed -e 's/v//; s/-rc/~rc/; s/-g/+git/; s/-/.post/; s/-/./' <<< "%src_version")
+Version:	%rpm_version
+Source0:	%{name}.tar.gz
+%endif
+
+%if %{with compat}
+Source1:	%{url}/download/v%{compat_version}/%{name}-%{compat_version}.tar.gz
+%endif
+
+%undefine __cmake_in_source_build
+
+%if %{?rhel}%{!?rhel:0} == 7
+%define cmake %cmake3
+%define __cmake %__cmake3
+%undefine __cmake3_in_source_build
+%endif
+
+%if %{with tests}
+# CppUnit crashes with LTO enabled
+%global _lto_cflags %nil
+%endif
+
+%if %{?rhel}%{!?rhel:0} == 7
+BuildRequires:	cmake3
+BuildRequires:	%{devtoolset}-toolchain
+%else
+BuildRequires:	cmake
+BuildRequires:	gcc-c++
+%endif
+BuildRequires:	gdb
+BuildRequires:	make
+BuildRequires:	pkgconfig
+BuildRequires:	fuse-devel
+BuildRequires:	krb5-devel
+BuildRequires:	libcurl-devel
+BuildRequires:	tinyxml-devel
+BuildRequires:	libxml2-devel
+BuildRequires:	ncurses-devel
+BuildRequires:	perl-generators
+BuildRequires:	readline-devel
+BuildRequires:	zlib-devel
+BuildRequires:	selinux-policy-devel
+BuildRequires:	systemd-rpm-macros
+BuildRequires:	systemd-devel
+%if %{?fedora}%{!?fedora:0} || %{?rhel}%{!?rhel:0} >= 8
+BuildRequires:	python3-devel
+BuildRequires:	python3-pip
+BuildRequires:	python3-setuptools
+BuildRequires:	python3-wheel
+%endif
+%if %{?rhel}%{!?rhel:0} == 7
+BuildRequires:	python2-devel
+BuildRequires:	python2-pip
+BuildRequires:	python2-setuptools
+BuildRequires:	python%{python3_pkgversion}-devel
+BuildRequires:	python%{python3_pkgversion}-pip
+BuildRequires:	python%{python3_pkgversion}-setuptools
+BuildRequires:	python%{python3_other_pkgversion}-devel
+BuildRequires:	python%{python3_other_pkgversion}-pip
+BuildRequires:	python%{python3_other_pkgversion}-setuptools
+%endif
+BuildRequires:	json-c-devel
+BuildRequires:	libmacaroons-devel
+BuildRequires:	libuuid-devel
+BuildRequires:	voms-devel
+BuildRequires:	scitokens-cpp-devel
+BuildRequires:	davix-devel
+
+%if %{with asan}
+BuildRequires:	libasan
+%if %{?rhel}%{!?rhel:0} == 7
+BuildRequires:	%{devtoolset}-libasan-devel
+%endif
+Requires:	libasan
+%endif
+
+%if %{with ceph}
+BuildRequires:	librados-devel
+BuildRequires:	libradosstriper-devel
+%endif
+
+%if %{with clang}
+%if %{?rhel}%{!?rhel:0} == 7
+BuildRequires:	%{llvmtoolset}-clang
+%else
+BuildRequires:	clang
+%endif
+%endif
+
+%if %{with docs}
+BuildRequires:	doxygen
+BuildRequires:	graphviz
+%{?el7:BuildRequires:	python2-sphinx}
+%{!?el7:BuildRequires:	python3-sphinx}
+%endif
+
+%if %{with openssl11}
+BuildRequires:	openssl11-devel
+%else
+BuildRequires:	openssl-devel
+%endif
+
+%if %{with tests}
+BuildRequires:	attr
+BuildRequires:	cppunit-devel
+BuildRequires:	gtest-devel
+BuildRequires:	openssl
+%endif
+
+%if %{with xrdec}
+BuildRequires:	isa-l-devel
+%endif
+
+Requires:	%{name}-client%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-server%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-selinux = %{epoch}:%{version}-%{release}
+
+%description
+The Extended root file server consists of a file server called xrootd
+and a cluster management server called cmsd.
+
+The xrootd server was developed for the root analysis framework to
+serve root files. However, the server is agnostic to file types and
+provides POSIX-like access to any type of file.
+
+The cmsd server is the next generation version of the olbd server,
+originally developed to cluster and load balance Objectivity/DB AMS
+database servers. It provides enhanced capability along with lower
+latency and increased throughput.
+
+%package server
+Summary:	XRootD server daemons
+Group:		System Environment/Daemons
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-server-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	expect
+Requires:	logrotate
+Requires(pre):	shadow-utils
+%{?systemd_requires}
+
+%description server
+This package contains the XRootD servers without the SELinux support.
+Unless you are installing on a system without SELinux also install the
+xrootd-selinux package.
+
+%package selinux
+Summary:	SELinux policy modules for the XRootD servers
+Group:		System Environment/Base
+BuildArch:	noarch
+Requires:	selinux-policy
+Requires(post):		policycoreutils
+Requires(postun):	policycoreutils
+
+%description selinux
+This package contains SELinux policy modules for the xrootd-server package.
+
+%package libs
+Summary:	Libraries used by XRootD servers and clients
+Group:		System Environment/Libraries
+
+%description libs
+This package contains libraries used by the XRootD servers and clients.
+
+%package devel
+Summary:	Development files for XRootD
+Group:		Development/Libraries
+Provides:	%{name}-libs-devel = %{epoch}:%{version}-%{release}
+Provides:	%{name}-libs-devel%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Obsoletes:	%{name}-libs-devel < %{epoch}:%{version}-%{release}
+
+%description devel
+This package contains header files and development libraries for XRootD
+development.
+
+%package client-libs
+Summary:	Libraries used by XRootD clients
+Group:		System Environment/Libraries
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description client-libs
+This package contains libraries used by XRootD clients.
+
+%package client-devel
+Summary:	Development files for XRootD clients
+Group:		Development/Libraries
+Provides:	%{name}-cl-devel = %{epoch}:%{version}-%{release}
+Provides:	%{name}-cl-devel%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-devel%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Obsoletes:	%{name}-cl-devel < %{epoch}:%{version}-%{release}
+
+%description client-devel
+This package contains header files and development libraries for XRootD
+client development.
+
+%package server-libs
+Summary:	Libraries used by XRootD servers
+Group:		System Environment/Libraries
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description server-libs
+This package contains libraries used by XRootD servers.
+
+%package server-devel
+Summary:	Development files for XRootD servers
+Group:		Development/Libraries
+Requires:	%{name}-devel%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-devel%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-server-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description server-devel
+This package contains header files and development libraries for XRootD
+server development.
+
+%package private-devel
+Summary:	Private XRootD headers
+Group:		Development/Libraries
+Requires:	%{name}-devel%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-devel%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-server-devel%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description private-devel
+This package contains some private XRootD headers. Backward and forward
+compatibility between versions is not guaranteed for these headers.
+
+%package client
+Summary:	XRootD command line client tools
+Group:		Applications/Internet
+Provides:	%{name}-cl = %{epoch}:%{version}-%{release}
+Provides:	%{name}-cl%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Obsoletes:	%{name}-cl < %{epoch}:%{version}-%{release}
+
+%description client
+This package contains the command line tools used to communicate with
+XRootD servers.
+
+%package fuse
+Summary:	XRootD FUSE tool
+Group:		Applications/Internet
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	fuse
+
+%description fuse
+This package contains the FUSE (file system in user space) XRootD mount
+tool.
+
+%package voms
+Summary:	VOMS attribute extractor plugin for XRootD
+Group:		System Environment/Libraries
+Provides:	vomsxrd = %{epoch}:%{version}-%{release}
+Provides:	%{name}-voms-plugin = %{epoch}:%{version}-%{release}
+Provides:	xrdhttpvoms = %{epoch}:%{version}-%{release}
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Obsoletes:	%{name}-voms-plugin < 1:0.6.0-3
+Obsoletes:	xrdhttpvoms < 0.2.5-9
+Obsoletes:	vomsxrd < 1:0.6.0-4
+
+%description voms
+The VOMS attribute extractor plugin for XRootD.
+
+%package scitokens
+Summary:	SciTokens authorization support for XRootD
+Group:		System Environment/Libraries
+License:	Apache-2.0 AND BSD-2-Clause AND BSD-3-Clause
+Requires:	%{name}-server%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description scitokens
+This ACC (authorization) plugin for the XRootD framework utilizes the
+SciTokens library to validate and extract authorization claims from a
+SciToken passed during a transfer. Configured appropriately, this
+allows the XRootD server admin to delegate authorization decisions for
+a subset of the namespace to an external issuer.
+
+%package -n xrdcl-http
+Summary:	HTTP client plugin for XRootD
+Group:		System Environment/Libraries
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description -n xrdcl-http
+xrdcl-http is an XRootD client plugin which allows XRootD to interact
+with HTTP repositories.
+
+%if %{with ceph}
+%package ceph
+Summary:	XRootD plugin for interfacing with the Ceph storage platform
+Group:		System Environment/Libraries
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description ceph
+The xrootd-ceph is an OSS layer plugin for the XRootD server for
+interfacing with the Ceph storage platform.
+%endif
+
+%if %{?rhel}%{!?rhel:0} == 7
+%package -n python2-%{name}
+Summary:	Python 2 bindings for XRootD
+Group:		System Environment/Libraries
+%py_provides	python2-%{name}
+Provides:	%{name}-python = %{epoch}:%{version}-%{release}
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Obsoletes:	%{name}-python < %{epoch}:%{version}-%{release}
+
+%description -n python2-%{name}
+This package contains Python 2 bindings for XRootD.
+%endif
+
+%package -n python%{python3_pkgversion}-%{name}
+Summary:	Python 3 bindings for XRootD
+Group:		System Environment/Libraries
+%py_provides	python%{python3_pkgversion}-%{name}
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description -n python%{python3_pkgversion}-%{name}
+This package contains Python 3 bindings for XRootD.
+
+%if %{?rhel}%{!?rhel:0} == 7
+%package -n python%{?python3_other_pkgversion}-%{name}
+Summary:	Python 3 bindings for XRootD
+Group:		System Environment/Libraries
+%py_provides	python%{python3_other_pkgversion}-%{name}
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+Requires:	%{name}-client-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description -n python%{?python3_other_pkgversion}-%{name}
+This package contains Python 3 bindings for XRootD.
+%endif
+
+%package doc
+Summary:	Developer documentation for the XRootD libraries
+Group:		Documentation
+BuildArch:	noarch
+
+%description doc
+This package contains the API documentation of the XRootD libraries.
+
+%if %{with compat}
+%package client-compat
+Summary:	XRootD 4 compatibility client libraries
+Group:		System Environment/Libraries
+
+%description client-compat
+This package contains compatibility libraries for XRootD 4 clients.
+
+%package server-compat
+Summary:	XRootD 4 compatibility server binaries
+Group:		System Environment/Daemons
+Requires:	%{name}-libs%{?_isa} = %{epoch}:%{version}-%{release}
+
+%description server-compat
+This package contains compatibility binaries for XRootD 4 servers.
+%endif
+
+%prep
+%if %{with compat}
+%setup -T -b 1 -n %{name}-%{compat_version}
+%endif
+
+%if %{with git}
+%autosetup -n %{name}
+%else
+%autosetup -p1
+%endif
+
+%build
+%if %{?rhel}%{!?rhel:0} == 7
+. /opt/rh/%{devtoolset}/enable
+%if %{with clang}
+. /opt/rh/%{llvmtoolset}/enable
+%endif
+%endif
+
+%if %{with clang}
+export CC=clang
+export CXX=clang++
+%endif
+
+%if %{with compat}
+%__cmake \
+    -S %{_builddir}/%{name}-%{compat_version} \
+    -B %{_builddir}/%{name}-%{compat_version}/build \
+%if %{with openssl11}
+    -DOPENSSL_INCLUDE_DIR=/usr/include/openssl11 \
+    -DOPENSSL_CRYPTO_LIBRARY=/usr/lib64/libcrypto.so.1.1 \
+    -DOPENSSL_SSL_LIBRARY=/usr/lib64/libssl.so.1.1 \
+%endif
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_C_FLAGS_RELEASE:STRING='%{optflags}' \
+    -DCMAKE_CXX_FLAGS_RELEASE:STRING='%{optflags}' \
+    -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
+    -DCMAKE_INSTALL_PREFIX:PATH=%{_prefix} \
+    -DINCLUDE_INSTALL_DIR:PATH=%{_includedir} \
+    -DCMAKE_INSTALL_LIBDIR:PATH=%{_libdir} \
+    -DCMAKE_INSTALL_SYSCONFDIR:PATH=%{_sysconfdir} \
+    -DFORCE_ENABLED:BOOL=TRUE \
+    -DENABLE_ASAN:BOOL=%{with asan} \
+    -DENABLE_FUSE:BOOL=TRUE \
+    -DENABLE_KRB5:BOOL=TRUE \
+    -DENABLE_MACAROONS:BOOL=TRUE \
+    -DENABLE_READLINE:BOOL=TRUE \
+    -DENABLE_SCITOKENS:BOOL=TRUE \
+    -DENABLE_TESTS:BOOL=FALSE \
+    -DENABLE_VOMS:BOOL=TRUE \
+    -DENABLE_XRDCL:BOOL=TRUE \
+    -DENABLE_XRDCLHTTP:BOOL=TRUE \
+    -DXRDCEPH_SUBMODULE:BOOL=%{with ceph} \
+    -DENABLE_XRDCLHTTP:BOOL=TRUE \
+    -DXRDCL_ONLY:BOOL=FALSE \
+    -DXRDCL_LIB_ONLY:BOOL=FALSE \
+    -DENABLE_PYTHON:BOOL=FALSE
+make -C %{_builddir}/%{name}-%{compat_version}/build %{?_smp_mflags}
+%endif
+
+%cmake \
+    -DFORCE_ENABLED:BOOL=TRUE \
+    -DUSE_SYSTEM_ISAL:BOOL=TRUE \
+    -DENABLE_ASAN:BOOL=%{with asan} \
+    -DENABLE_FUSE:BOOL=TRUE \
+    -DENABLE_KRB5:BOOL=TRUE \
+    -DENABLE_MACAROONS:BOOL=TRUE \
+    -DENABLE_READLINE:BOOL=TRUE \
+    -DENABLE_SCITOKENS:BOOL=TRUE \
+    -DENABLE_TESTS:BOOL=%{with tests} \
+    -DENABLE_VOMS:BOOL=TRUE \
+    -DENABLE_XRDCL:BOOL=TRUE \
+    -DENABLE_XRDCLHTTP:BOOL=TRUE \
+    -DENABLE_XRDEC:BOOL=%{with xrdec} \
+    -DXRDCEPH_SUBMODULE:BOOL=%{with ceph} \
+    -DENABLE_XRDCLHTTP:BOOL=TRUE \
+    -DXRDCL_ONLY:BOOL=FALSE \
+    -DXRDCL_LIB_ONLY:BOOL=FALSE \
+%if %{with openssl11}
+    -DOPENSSL_INCLUDE_DIR=/usr/include/openssl11 \
+    -DOPENSSL_CRYPTO_LIBRARY=/usr/lib64/libcrypto.so.1.1 \
+    -DOPENSSL_SSL_LIBRARY=/usr/lib64/libssl.so.1.1 \
+%endif
+    -DENABLE_PYTHON:BOOL=TRUE \
+    -DINSTALL_PYTHON_BINDINGS:BOOL=FALSE \
+%if %{?rhel}%{!?rhel:0} == 7
+    -DXRD_PYTHON_REQ_VERSION=%{python2_version}
+%else
+    -DXRD_PYTHON_REQ_VERSION=%{python3_version}
+%endif
+
+%cmake3_build
+
+make -C packaging/common -f /usr/share/selinux/devel/Makefile
+
+%if %{with docs}
+doxygen Doxyfile
+%endif
+
+%if %{with tests}
+%check
+%ctest3
+%endif
+
+%install
+%if %{?rhel}%{!?rhel:0} == 7
+. /opt/rh/%{devtoolset}/enable
+%endif
+
+%if %{with compat}
+pushd %{_builddir}/%{name}-%{compat_version}/build
+
+make install DESTDIR=%{buildroot}
+
+rm -rf %{buildroot}%{_datadir} %{buildroot}%{_includedir}
+rm -f %{buildroot}%{_libdir}/libXrd{AppUtils,Cl,Client,Crypto,CryptoLite}.so
+rm -f %{buildroot}%{_libdir}/libXrd{Ffs,Main,Ofs,Posix*,Server,Utils}.so
+
+for i in cmsd frm_purged frm_xfrd xrootd; do
+	mv %{buildroot}%{_bindir}/$i %{buildroot}%{_bindir}/${i}-4
+done
+
+pushd %{buildroot}%{_bindir}
+find . ! -name '*-4' -delete
+popd
+
+popd
+%endif
+
+%cmake3_install
+
+# Remove test binaries and libraries
+%if %{with tests}
+	rm -f %{buildroot}%{_bindir}/test-runner
+	rm -f %{buildroot}%{_bindir}/xrdshmap
+	rm -f %{buildroot}%{_libdir}/libXrd*Tests*
+	rm -f %{buildroot}%{_libdir}/libXrdClTestMonitor*.so
+%endif
+
+%if %{with ceph}
+	rm -f %{buildroot}%{_libdir}/libXrdCephPosix.so
+%endif
+
+rm -f %{buildroot}%{python3_sitearch}/xrootd-*.*-info/direct_url.json
+rm -f %{buildroot}%{python3_sitearch}/xrootd-*.*-info/RECORD
+[ -r %{buildroot}%{python3_sitearch}/xrootd-*.*-info/INSTALLER ] && \
+	sed s/pip/rpm/ -i %{buildroot}%{python3_sitearch}/xrootd-*.*-info/INSTALLER
+
+%{__python3} -m pip install \
+	--no-deps --ignore-installed --disable-pip-version-check --verbose \
+	--prefix %{buildroot}%{_prefix} %{_vpath_builddir}/bindings/python
+
+%if %{?rhel}%{!?rhel:0} == 7
+%{__python2} -m pip install \
+	--no-deps --ignore-installed --disable-pip-version-check --verbose \
+	--prefix %{buildroot}%{_prefix} %{_vpath_builddir}/bindings/python
+
+%{__python3_other} -m pip install \
+	--no-deps --ignore-installed --disable-pip-version-check --verbose \
+	--prefix %{buildroot}%{_prefix} %{_vpath_builddir}/bindings/python
+%endif
+
+%if %{with docs}
+%if %{?rhel}%{!?rhel:0} == 7
+LD_LIBRARY_PATH=%{buildroot}%{_libdir} \
+PYTHONPATH=%{buildroot}%{python2_sitearch} \
+PYTHONDONTWRITEBYTECODE=1 \
+make -C bindings/python/docs html
+%endif
+%if %{?fedora}%{!?fedora:0} || %{?rhel}%{!?rhel:0} >= 8
+LD_LIBRARY_PATH=%{buildroot}%{_libdir} \
+PYTHONPATH=%{buildroot}%{python3_sitearch} \
+PYTHONDONTWRITEBYTECODE=1 \
+make -C bindings/python/docs html SPHINXBUILD=sphinx-build-3
+%endif
+%endif
+
+# Service unit files
+mkdir -p %{buildroot}%{_unitdir}
+install -m 644 packaging/common/xrootd@.service %{buildroot}%{_unitdir}
+install -m 644 packaging/common/xrootd@.socket %{buildroot}%{_unitdir}
+install -m 644 packaging/common/xrdhttp@.socket %{buildroot}%{_unitdir}
+install -m 644 packaging/common/cmsd@.service %{buildroot}%{_unitdir}
+install -m 644 packaging/common/frm_xfrd@.service %{buildroot}%{_unitdir}
+install -m 644 packaging/common/frm_purged@.service %{buildroot}%{_unitdir}
+mkdir -p %{buildroot}%{_tmpfilesdir}
+install -m 644 packaging/rhel/xrootd.tmpfiles %{buildroot}%{_tmpfilesdir}/%{name}.conf
+
+# Server config
+mkdir -p %{buildroot}%{_sysconfdir}/%{name}
+install -m 644 -p packaging/common/%{name}-clustered.cfg \
+	%{buildroot}%{_sysconfdir}/%{name}/%{name}-clustered.cfg
+install -m 644 -p packaging/common/%{name}-standalone.cfg \
+	%{buildroot}%{_sysconfdir}/%{name}/%{name}-standalone.cfg
+install -m 644 -p packaging/common/%{name}-filecache-clustered.cfg \
+	%{buildroot}%{_sysconfdir}/%{name}/%{name}-filecache-clustered.cfg
+install -m 644 -p packaging/common/%{name}-filecache-standalone.cfg \
+	%{buildroot}%{_sysconfdir}/%{name}/%{name}-filecache-standalone.cfg
+sed 's!/usr/lib64/!!' packaging/common/%{name}-http.cfg > \
+	%{buildroot}%{_sysconfdir}/%{name}/%{name}-http.cfg
+
+# Client config
+mkdir -p %{buildroot}%{_sysconfdir}/%{name}/client.plugins.d
+install -m 644 -p packaging/common/client.conf \
+	%{buildroot}%{_sysconfdir}/%{name}/client.conf
+sed 's!/usr/lib/!!' packaging/common/client-plugin.conf.example > \
+	%{buildroot}%{_sysconfdir}/%{name}/client.plugins.d/client-plugin.conf.example
+sed -e 's!/usr/lib64/!!' -e 's!-5!!' packaging/common/recorder.conf > \
+	%{buildroot}%{_sysconfdir}/%{name}/client.plugins.d/recorder.conf
+sed 's!/usr/lib64/!!' packaging/common/http.client.conf.example > \
+	%{buildroot}%{_sysconfdir}/%{name}/client.plugins.d/xrdcl-http-plugin.conf
+
+chmod 644 %{buildroot}%{_datadir}/%{name}/utils/XrdCmsNotify.pm
+
+sed 's!/usr/bin/env perl!/usr/bin/perl!' -i \
+	%{buildroot}%{_datadir}/%{name}/utils/netchk \
+	%{buildroot}%{_datadir}/%{name}/utils/XrdCmsNotify.pm \
+	%{buildroot}%{_datadir}/%{name}/utils/XrdOlbMonPerf
+
+sed 's!/usr/bin/env bash!/bin/bash!' -i %{buildroot}%{_bindir}/xrootd-config
+
+mkdir -p %{buildroot}%{_sysconfdir}/%{name}/config.d
+
+mkdir -p %{buildroot}%{_localstatedir}/log/%{name}
+mkdir -p %{buildroot}%{_localstatedir}/spool/%{name}
+
+mkdir -p %{buildroot}%{_sysconfdir}/logrotate.d
+install -m 644 -p packaging/common/%{name}.logrotate \
+	%{buildroot}%{_sysconfdir}/logrotate.d/%{name}
+
+mkdir -p %{buildroot}%{_datadir}/selinux/packages/%{name}
+install -m 644 -p packaging/common/%{name}.pp \
+	%{buildroot}%{_datadir}/selinux/packages/%{name}
+
+%if %{with docs}
+	mkdir -p %{buildroot}%{_pkgdocdir}
+	cp -pr doxydoc/html %{buildroot}%{_pkgdocdir}
+
+	cp -pr bindings/python/docs/build/html %{buildroot}%{_pkgdocdir}/python
+	rm %{buildroot}%{_pkgdocdir}/python/.buildinfo
+%endif
+
+%ldconfig_scriptlets libs
+
+%ldconfig_scriptlets client-libs
+
+%ldconfig_scriptlets server-libs
+
+%pre server
+getent group %{name} >/dev/null || groupadd -r %{name}
+getent passwd %{name} >/dev/null || useradd -r -g %{name} -s /sbin/nologin \
+	-d %{_localstatedir}/spool/%{name} -c "System user for XRootD" %{name}
+
+%post server
+%tmpfiles_create %{_tmpfilesdir}/%{name}.conf
+
+if [ $1 -eq 1 ] ; then
+	systemctl daemon-reload >/dev/null 2>&1 || :
+fi
+
+%preun server
+if [ $1 -eq 0 ] ; then
+	for DAEMON in xrootd cmsd frm_purged frm_xfrd; do
+		for INSTANCE in `systemctl | grep $DAEMON@ | awk '{print $1;}'`; do
+			systemctl --no-reload disable $INSTANCE > /dev/null 2>&1 || :
+			systemctl stop $INSTANCE > /dev/null 2>&1 || :
+		done
+	done
+fi
+
+%postun server
+if [ $1 -ge 1 ] ; then
+	systemctl daemon-reload >/dev/null 2>&1 || :
+	for DAEMON in xrootd cmsd frm_purged frm_xfrd; do
+		for INSTANCE in `systemctl | grep $DAEMON@ | awk '{print $1;}'`; do
+			systemctl try-restart $INSTANCE >/dev/null 2>&1 || :
+		done
+	done
+fi
+
+%post selinux
+semodule -i %{_datadir}/selinux/packages/%{name}/%{name}.pp >/dev/null 2>&1 || :
+
+%postun selinux
+if [ $1 -eq 0 ] ; then
+	semodule -r %{name} >/dev/null 2>&1 || :
+fi
+
+%files
+# Empty
+
+%files server
+%{_bindir}/cconfig
+%{_bindir}/cmsd
+%{_bindir}/frm_admin
+%{_bindir}/frm_purged
+%{_bindir}/frm_xfragent
+%{_bindir}/frm_xfrd
+%{_bindir}/mpxstats
+%{_bindir}/wait41
+%{_bindir}/xrdacctest
+%{_bindir}/xrdpfc_print
+%{_bindir}/xrdpwdadmin
+%{_bindir}/xrdsssadmin
+%{_bindir}/xrootd
+%{_mandir}/man8/cmsd.8*
+%{_mandir}/man8/frm_admin.8*
+%{_mandir}/man8/frm_purged.8*
+%{_mandir}/man8/frm_xfragent.8*
+%{_mandir}/man8/frm_xfrd.8*
+%{_mandir}/man8/mpxstats.8*
+%{_mandir}/man8/xrdpfc_print.8*
+%{_mandir}/man8/xrdpwdadmin.8*
+%{_mandir}/man8/xrdsssadmin.8*
+%{_mandir}/man8/xrootd.8*
+%dir %{_datadir}/%{name}
+%{_datadir}/%{name}/utils
+%{_unitdir}/*
+%{_tmpfilesdir}/%{name}.conf
+%config(noreplace) %{_sysconfdir}/logrotate.d/%{name}
+%dir %{_sysconfdir}/%{name}/config.d
+%attr(-,xrootd,xrootd) %config(noreplace) %{_sysconfdir}/%{name}/*.cfg
+%attr(-,xrootd,xrootd) %{_localstatedir}/log/%{name}
+%attr(-,xrootd,xrootd) %{_localstatedir}/spool/%{name}
+%ghost %attr(-,xrootd,xrootd) %{_rundir}/%{name}
+
+%files selinux
+%{_datadir}/selinux/packages/%{name}/%{name}.pp
+
+%files libs
+%{_libdir}/libXrdAppUtils.so.*
+%{_libdir}/libXrdCrypto.so.*
+%{_libdir}/libXrdCryptoLite.so.*
+%{_libdir}/libXrdUtils.so.*
+%{_libdir}/libXrdXml.so.*
+# Plugins
+%{_libdir}/libXrdCksCalczcrc32-5.so
+%{_libdir}/libXrdCryptossl-5.so
+%{_libdir}/libXrdSec-5.so
+%{_libdir}/libXrdSecProt-5.so
+%{_libdir}/libXrdSecgsi-5.so
+%{_libdir}/libXrdSecgsiAUTHZVO-5.so
+%{_libdir}/libXrdSecgsiGMAPDN-5.so
+%{_libdir}/libXrdSeckrb5-5.so
+%{_libdir}/libXrdSecpwd-5.so
+%{_libdir}/libXrdSecsss-5.so
+%{_libdir}/libXrdSecunix-5.so
+%{_libdir}/libXrdSecztn-5.so
+%license COPYING* LICENSE
+
+%files devel
+%{_bindir}/xrootd-config
+%dir %{_includedir}/%{name}
+%{_includedir}/%{name}/XProtocol
+%{_includedir}/%{name}/Xrd
+%{_includedir}/%{name}/XrdCks
+%{_includedir}/%{name}/XrdNet
+%{_includedir}/%{name}/XrdOuc
+%{_includedir}/%{name}/XrdSec
+%{_includedir}/%{name}/XrdSys
+%{_includedir}/%{name}/XrdXml
+%{_includedir}/%{name}/XrdVersion.hh
+%{_libdir}/libXrdAppUtils.so
+%{_libdir}/libXrdCrypto.so
+%{_libdir}/libXrdCryptoLite.so
+%{_libdir}/libXrdUtils.so
+%{_libdir}/libXrdXml.so
+%{_libdir}/cmake/XRootD
+%dir %{_datadir}/%{name}
+
+%files client-libs
+%{_libdir}/libXrdCl.so.*
+%if %{with xrdec}
+%{_libdir}/libXrdEc.so.*
+%endif
+%{_libdir}/libXrdFfs.so.*
+%{_libdir}/libXrdPosix.so.*
+%{_libdir}/libXrdPosixPreload.so.*
+# This lib may be used for LD_PRELOAD so the .so link needs to be included
+%{_libdir}/libXrdPosixPreload.so
+%{_libdir}/libXrdSsiLib.so.*
+%{_libdir}/libXrdSsiShMap.so.*
+# Plugins
+%{_libdir}/libXrdClProxyPlugin-5.so
+%{_libdir}/libXrdClRecorder-5.so
+%dir %{_sysconfdir}/%{name}
+%config(noreplace) %{_sysconfdir}/%{name}/client.conf
+%dir %{_sysconfdir}/%{name}/client.plugins.d
+%config(noreplace) %{_sysconfdir}/%{name}/client.plugins.d/client-plugin.conf.example
+%config(noreplace) %{_sysconfdir}/%{name}/client.plugins.d/recorder.conf
+
+%files client-devel
+%{_includedir}/%{name}/XrdCl
+%{_includedir}/%{name}/XrdPosix
+%{_libdir}/libXrdCl.so
+%if %{with xrdec}
+%{_libdir}/libXrdEc.so
+%endif
+%{_libdir}/libXrdFfs.so
+%{_libdir}/libXrdPosix.so
+
+%files server-libs
+%{_libdir}/libXrdHttpUtils.so.*
+%{_libdir}/libXrdServer.so.*
+# Plugins
+%{_libdir}/libXrdBlacklistDecision-5.so
+%{_libdir}/libXrdBwm-5.so
+%{_libdir}/libXrdCmsRedirectLocal-5.so
+%{_libdir}/libXrdFileCache-5.so
+%{_libdir}/libXrdHttp-5.so
+%{_libdir}/libXrdHttpTPC-5.so
+%{_libdir}/libXrdMacaroons-5.so
+%{_libdir}/libXrdN2No2p-5.so
+%{_libdir}/libXrdOfsPrepGPI-5.so
+%{_libdir}/libXrdOssCsi-5.so
+%{_libdir}/libXrdOssSIgpfsT-5.so
+%{_libdir}/libXrdPfc-5.so
+%{_libdir}/libXrdPss-5.so
+%{_libdir}/libXrdSsi-5.so
+%{_libdir}/libXrdSsiLog-5.so
+%{_libdir}/libXrdThrottle-5.so
+%{_libdir}/libXrdXrootd-5.so
+
+%files server-devel
+%{_includedir}/%{name}/XrdAcc
+%{_includedir}/%{name}/XrdCms
+%{_includedir}/%{name}/XrdHttp
+%{_includedir}/%{name}/XrdOfs
+%{_includedir}/%{name}/XrdOss
+%{_includedir}/%{name}/XrdPfc
+%{_includedir}/%{name}/XrdSfs
+%{_includedir}/%{name}/XrdXrootd
+%{_libdir}/libXrdHttpUtils.so
+%{_libdir}/libXrdServer.so
+
+%files private-devel
+%{_includedir}/%{name}/private
+%{_libdir}/libXrdSsiLib.so
+%{_libdir}/libXrdSsiShMap.so
+
+%files client
+%{_bindir}/xrdadler32
+%{_bindir}/xrdcks
+%{_bindir}/xrdcopy
+%{_bindir}/xrdcp
+%{_bindir}/xrdcrc32c
+%{_bindir}/xrdfs
+%{_bindir}/xrdgsiproxy
+%{_bindir}/xrdgsitest
+%{_bindir}/xrdmapc
+%{_bindir}/xrdpinls
+%{_bindir}/xrdreplay
+%{_mandir}/man1/xrdadler32.1*
+%{_mandir}/man1/xrdcopy.1*
+%{_mandir}/man1/xrdcp.1*
+%{_mandir}/man1/xrdfs.1*
+%{_mandir}/man1/xrdgsiproxy.1*
+%{_mandir}/man1/xrdgsitest.1*
+%{_mandir}/man1/xrdmapc.1*
+
+%files fuse
+%{_bindir}/xrootdfs
+%{_mandir}/man1/xrootdfs.1*
+
+%files voms
+%{_libdir}/libXrdVoms-5.so
+%{_libdir}/libXrdHttpVOMS-5.so
+%{_libdir}/libXrdSecgsiVOMS-5.so
+%doc %{_mandir}/man1/libXrdVoms.1*
+%doc %{_mandir}/man1/libXrdSecgsiVOMS.1*
+
+%files scitokens
+%{_libdir}/libXrdAccSciTokens-5.so
+%doc src/XrdSciTokens/README.md
+
+%files -n xrdcl-http
+%{_libdir}/libXrdClHttp-5.so
+%config(noreplace) %{_sysconfdir}/%{name}/client.plugins.d/xrdcl-http-plugin.conf
+
+%if %{with ceph}
+%files ceph
+%{_libdir}/libXrdCeph-5.so
+%{_libdir}/libXrdCephXattr-5.so
+%{_libdir}/libXrdCephPosix.so.*
+%endif
+
+%files -n python%{python3_pkgversion}-%{name}
+%{python3_sitearch}/xrootd-*.*-info
+%{python3_sitearch}/pyxrootd
+%{python3_sitearch}/XRootD
+
+%if %{?rhel}%{!?rhel:0} == 7
+%files -n python2-%{name}
+%{python2_sitearch}/xrootd-*.*-info
+%{python2_sitearch}/pyxrootd
+%{python2_sitearch}/XRootD
+
+%files -n python%{?python3_other_pkgversion}-%{name}
+%{python3_other_sitearch}/xrootd-*.*-info
+%{python3_other_sitearch}/pyxrootd
+%{python3_other_sitearch}/XRootD
+%endif
+
+%if %{with docs}
+%files doc
+%doc %{_pkgdocdir}
+%endif
+
+%if %{with compat}
+%files client-compat
+# from xrootd-libs:
+%{_libdir}/libXrdAppUtils.so.1*
+%{_libdir}/libXrdCks*-4.so
+%{_libdir}/libXrdClProxyPlugin-4.so
+%{_libdir}/libXrdCrypto.so.1*
+%{_libdir}/libXrdCryptoLite.so.1*
+%{_libdir}/libXrdCryptossl-4.so
+%{_libdir}/libXrdSec*-4.so
+%{_libdir}/libXrdUtils.so.2*
+%{_libdir}/libXrdXml.so.2*
+
+# from xrootd-client-libs
+%{_libdir}/libXrdCl.so.2*
+%{_libdir}/libXrdClient.so.2*
+%{_libdir}/libXrdFfs.so.2*
+%{_libdir}/libXrdPosix.so.2*
+%{_libdir}/libXrdPosixPreload.so.1*
+%{_libdir}/libXrdSsiLib.so.1*
+%{_libdir}/libXrdSsiShMap.so.1*
+
+%files server-compat
+# from server (renamed)
+%{_bindir}/cmsd-4
+%{_bindir}/frm_purged-4
+%{_bindir}/frm_xfrd-4
+%{_bindir}/xrootd-4
+# from server-libs
+%{_libdir}/libXrdBwm-4.so
+%{_libdir}/libXrdPss-4.so
+%{_libdir}/libXrdXrootd-4.so
+%{_libdir}/libXrdFileCache-4.so
+%{_libdir}/libXrdBlacklistDecision-4.so
+%{_libdir}/libXrdHttp-4.so
+%{_libdir}/libXrdHttpTPC-4.so
+%{_libdir}/libXrdHttpUtils.so.1*
+%{_libdir}/libXrdMacaroons-4.so
+%{_libdir}/libXrdN2No2p-4.so
+%{_libdir}/libXrdOssSIgpfsT-4.so
+%{_libdir}/libXrdServer.so.2*
+%{_libdir}/libXrdSsi-4.so
+%{_libdir}/libXrdSsiLog-4.so
+%{_libdir}/libXrdThrottle-4.so
+%{_libdir}/libXrdCmsRedirectLocal-4.so
+%{_libdir}/libXrdVoms-4.so
+%endif
+
+%changelog
+
+* Fri Mar 08 2024 Guilherme Amadio  - 1:5.6.9-1
+- XRootD 5.6.9
+
+* Fri Feb 23 2024 Guilherme Amadio  - 1:5.6.8-1
+- XRootD 5.6.8
+
+* Tue Feb 06 2024 Guilherme Amadio  - 1:5.6.7-1
+- XRootD 5.6.7
+
+* Thu Jan 25 2024 Guilherme Amadio  - 1:5.6.6-1
+- XRootD 5.6.6
+
+* Mon Jan 22 2024 Guilherme Amadio  - 1:5.6.5-1
+- XRootD 5.6.5
+
+* Fri Dec 08 2023 Guilherme Amadio  - 1:5.6.4-1
+- Use isa-l library from the system
+- Extract version from tarball when building git snapshots
+- XRootD 5.6.4
+
+* Fri Oct 27 2023 Guilherme Amadio  - 1:5.6.3-2
+- XRootD 5.6.3
+
+* Mon Sep 18 2023 Guilherme Amadio  - 1:5.6.2-2
+- Add patch with fix for id parsing in XrdAccAuthFile (#2088)
+
+* Fri Sep 15 2023 Guilherme Amadio  - 1:5.6.2-1
+- Link XRootD 4 with openssl1.1 when using --with openssl11
+- XRootD 5.6.2
+
+* Fri Aug 11 2023 Guilherme Amadio  - 1:5.6.1-1
+- Modernize spec file to add more optional features and select
+  default build options automatically for each supported OS.
+- Use latest official release tarball by default.
+- Enable snapshot builds from git.
+
+* Thu Oct 15 2020 Michal Simon  - 5.0.2-1
+- Introduce xrootd-scitokens package
+
+* Wed May 27 2020 Michal Simon  - 4.12.2-1
+- Remove xrootd-voms-devel
+
+* Fri Apr 17 2020 Michal Simon  - 4.12.0-1
+- Introduce xrootd-voms and xrootd-voms-devel packages
+
+* Mon Sep 02 2019 Michal Simon  - 4.10.1-1
+- Move xrdmapc to client package
+
+* Fri Aug 30 2019 Michal Simon  - 5.0.0
+- Remove XRootD 3.x.x compat package
+
+* Wed Apr 17 2019 Michal Simon  - 4.10.0-1
+- Create add xrdcl-http package
+
+* Tue Jan 08 2019 Edgar Fajardo 
+- Create config dir /etc/xrootd/config.d
+
+* Tue May 08 2018 Michal Simon 
+- Make python3 sub-package optional
+
+* Fri Nov 10 2017 Michal Simon  - 1:4.8.0-1
+- Add python3 sub-package
+- Rename python sub-package
+
+* Tue Dec 13 2016 Gerardo Ganis 
+- Add xrdgsitest to xrootd-client-devel
+
+* Mon Mar 16 2015 Lukasz Janyst 
+- create the python package
+
+* Wed Mar 11 2015 Lukasz Janyst 
+- create the xrootd-ceph package
+
+* Thu Oct 30 2014 Lukasz Janyst 
+- update for 4.1 and introduce 3.3.6 compat packages
+
+* Thu Aug 28 2014 Lukasz Janyst 
+- add support for systemd
+
+* Wed Aug 27 2014 Lukasz Janyst 
+- use generic selinux policy build mechanisms
+
+* Tue Apr 01 2014 Lukasz Janyst 
+- correct the license field (LGPLv3+)
+- rename to xrootd4
+- add 'conflicts' statements
+- remove 'provides' and 'obsoletes'
+
+* Mon Mar 31 2014 Lukasz Janyst 
+- Add selinux policy
+
+* Fri Jan 24 2014 Lukasz Janyst 
+- Import XrdHttp
+
+* Fri Jun 7 2013 Lukasz Janyst 
+- adopt the EPEL RPM layout by Mattias Ellert
+
+* Tue Apr 2 2013 Lukasz Janyst 
+- remove perl
+
+* Thu Nov 1 2012 Justin Salmon 
+- add tests package
+
+* Fri Oct 21 2011 Lukasz Janyst  3.1.0-1
+- bump the version to 3.1.0
+
+* Mon Apr 11 2011 Lukasz Janyst  3.0.3-1
+- the first RPM release - version 3.0.3
+- the detailed release notes are available at:
+  http://xrootd.org/download/ReleaseNotes.html