From 779f35c0f6cb383c3789b439a78d487393eab1a0 Mon Sep 17 00:00:00 2001 From: jennmald Date: Wed, 6 Nov 2024 11:39:35 -0500 Subject: [PATCH 1/4] adding np.nan and testing workflows --- src/blop/agent.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/blop/agent.py b/src/blop/agent.py index f5959689..79cd5309 100644 --- a/src/blop/agent.py +++ b/src/blop/agent.py @@ -523,7 +523,7 @@ def acquire(self, points): logging.warning(f"Error in acquisition/digestion: {repr(error)}") products = pd.DataFrame(points) for obj in self.objectives(active=True): - products.loc[:, obj.name] = np.nan + products.loc[:, obj.name] = torch.tensor(np.nan) if len(products) != n: raise ValueError("The table returned by the digestion function must be the same length as the sampled inputs!") @@ -923,7 +923,7 @@ def train_inputs(self, index=None, **subset_kwargs): # check that inputs values are inside acceptable values valid = (raw_inputs >= dof._trust_domain[0]) & (raw_inputs <= dof._trust_domain[1]) - raw_inputs = torch.where(valid, raw_inputs, np.nan) + raw_inputs = torch.where(valid, raw_inputs, torch.tensor(np.nan)) return dof._transform(raw_inputs) @@ -950,7 +950,7 @@ def train_targets(self, concatenate=False, **subset_kwargs): # check that targets values are inside acceptable values valid = (y >= obj._trust_domain[0]) & (y <= obj._trust_domain[1]) - y = torch.where(valid, y, np.nan) + y = torch.where(valid, y, torch.tensor(np.nan)) targets_dict[obj.name] = obj._transform(y) @@ -961,7 +961,7 @@ def train_targets(self, concatenate=False, **subset_kwargs): all_valid_mask &= ~values.isnan() for name in targets_dict.keys(): - targets_dict[name] = targets_dict[name].where(all_valid_mask, np.nan) + targets_dict[name] = targets_dict[name].where(all_valid_mask, torch.tensor(np.nan)) if concatenate: return torch.cat([values.unsqueeze(-1) for values in targets_dict.values()], axis=-1) From 3bffc20fc06ef7fb0ce3f6c14098c6b27406cb00 Mon Sep 17 00:00:00 2001 From: jennmald Date: Wed, 6 Nov 2024 14:40:01 -0500 Subject: [PATCH 2/4] add black formatting --- src/blop/agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blop/agent.py b/src/blop/agent.py index 79cd5309..32342b33 100644 --- a/src/blop/agent.py +++ b/src/blop/agent.py @@ -961,7 +961,7 @@ def train_targets(self, concatenate=False, **subset_kwargs): all_valid_mask &= ~values.isnan() for name in targets_dict.keys(): - targets_dict[name] = targets_dict[name].where(all_valid_mask, torch.tensor(np.nan)) + targets_dict[name] = targets_dict[name].where(all_valid_mask, torch.tensor(np.nan)) if concatenate: return torch.cat([values.unsqueeze(-1) for values in targets_dict.values()], axis=-1) From 9a7486c34dd6f92bbe4f52803f570c89df0e196d Mon Sep 17 00:00:00 2001 From: jennmald Date: Thu, 21 Nov 2024 10:10:40 -0500 Subject: [PATCH 3/4] update docstring for enforce_all_objectives --- src/blop/agent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/blop/agent.py b/src/blop/agent.py index 32342b33..968e63fd 100644 --- a/src/blop/agent.py +++ b/src/blop/agent.py @@ -101,6 +101,8 @@ def __init__( A databroker instance. verbose : bool To be verbose or not. + enforce_all_objectives_valid: bool + Whether to remove all beam diagnostics if one is bad. If `True`, disagnostics will be removed. tolerate_acquisition_errors : bool Whether to allow errors during acquistion. If `True`, errors will be caught as warnings. sample_center_on_init : bool From 5ad651172c8051f09eab53e5101cd7e1c867e7dd Mon Sep 17 00:00:00 2001 From: jennmald Date: Wed, 27 Nov 2024 14:53:48 -0500 Subject: [PATCH 4/4] merge main --- bltest/bin/Activate.ps1 | 247 ++ bltest/bin/activate | 70 + bltest/bin/activate.csh | 27 + bltest/bin/activate.fish | 69 + bltest/bin/bluesky-0MQ-proxy | 8 + bltest/bin/convert-caffe2-to-onnx | 8 + bltest/bin/convert-onnx-to-caffe2 | 8 + bltest/bin/cpuinfo | 8 + bltest/bin/dask | 8 + bltest/bin/f2py | 8 + bltest/bin/fonttools | 8 + bltest/bin/fs_rename | 85 + bltest/bin/imageio_download_bin | 8 + bltest/bin/imageio_remove_bin | 8 + bltest/bin/intake | 8 + bltest/bin/intake-server | 8 + bltest/bin/isympy | 8 + bltest/bin/jsonschema | 8 + bltest/bin/lsm2bin | 8 + bltest/bin/normalizer | 8 + bltest/bin/pint-convert | 8 + bltest/bin/pip | 8 + bltest/bin/pip3 | 8 + bltest/bin/pip3.12 | 8 + bltest/bin/pt2to3 | 8 + bltest/bin/ptdump | 8 + bltest/bin/ptrepack | 8 + bltest/bin/pttree | 8 + bltest/bin/pyftmerge | 8 + bltest/bin/pyftsubset | 8 + bltest/bin/python | 1 + bltest/bin/python3 | 1 + bltest/bin/python3.12 | 1 + bltest/bin/regenerate-schema | 8 + bltest/bin/start_md_server | 92 + bltest/bin/tiff2fsspec | 8 + bltest/bin/tiffcomment | 8 + bltest/bin/tifffile | 8 + bltest/bin/torchrun | 8 + bltest/bin/tqdm | 8 + bltest/bin/ttx | 8 + bltest/include/b2nd.h | 581 +++++ bltest/include/blosc2.h | 2608 ++++++++++++++++++++++ bltest/include/blosc2/blosc2-common.h | 80 + bltest/include/blosc2/blosc2-export.h | 48 + bltest/include/blosc2/blosc2-stdio.h | 117 + bltest/include/blosc2/codecs-registry.h | 50 + bltest/include/blosc2/filters-registry.h | 50 + bltest/include/blosc2/tuners-registry.h | 37 + bltest/pyvenv.cfg | 5 + bltest/share/man/man1/isympy.1 | 188 ++ bltest/share/man/man1/ttx.1 | 225 ++ src/blop/__init__.py | 2 +- src/blop/tests/test_sims.py | 33 + 54 files changed, 4872 insertions(+), 1 deletion(-) create mode 100644 bltest/bin/Activate.ps1 create mode 100644 bltest/bin/activate create mode 100644 bltest/bin/activate.csh create mode 100644 bltest/bin/activate.fish create mode 100755 bltest/bin/bluesky-0MQ-proxy create mode 100755 bltest/bin/convert-caffe2-to-onnx create mode 100755 bltest/bin/convert-onnx-to-caffe2 create mode 100755 bltest/bin/cpuinfo create mode 100755 bltest/bin/dask create mode 100755 bltest/bin/f2py create mode 100755 bltest/bin/fonttools create mode 100755 bltest/bin/fs_rename create mode 100755 bltest/bin/imageio_download_bin create mode 100755 bltest/bin/imageio_remove_bin create mode 100755 bltest/bin/intake create mode 100755 bltest/bin/intake-server create mode 100755 bltest/bin/isympy create mode 100755 bltest/bin/jsonschema create mode 100755 bltest/bin/lsm2bin create mode 100755 bltest/bin/normalizer create mode 100755 bltest/bin/pint-convert create mode 100755 bltest/bin/pip create mode 100755 bltest/bin/pip3 create mode 100755 bltest/bin/pip3.12 create mode 100755 bltest/bin/pt2to3 create mode 100755 bltest/bin/ptdump create mode 100755 bltest/bin/ptrepack create mode 100755 bltest/bin/pttree create mode 100755 bltest/bin/pyftmerge create mode 100755 bltest/bin/pyftsubset create mode 120000 bltest/bin/python create mode 120000 bltest/bin/python3 create mode 120000 bltest/bin/python3.12 create mode 100755 bltest/bin/regenerate-schema create mode 100755 bltest/bin/start_md_server create mode 100755 bltest/bin/tiff2fsspec create mode 100755 bltest/bin/tiffcomment create mode 100755 bltest/bin/tifffile create mode 100755 bltest/bin/torchrun create mode 100755 bltest/bin/tqdm create mode 100755 bltest/bin/ttx create mode 100644 bltest/include/b2nd.h create mode 100644 bltest/include/blosc2.h create mode 100644 bltest/include/blosc2/blosc2-common.h create mode 100644 bltest/include/blosc2/blosc2-export.h create mode 100644 bltest/include/blosc2/blosc2-stdio.h create mode 100644 bltest/include/blosc2/codecs-registry.h create mode 100644 bltest/include/blosc2/filters-registry.h create mode 100644 bltest/include/blosc2/tuners-registry.h create mode 100644 bltest/pyvenv.cfg create mode 100644 bltest/share/man/man1/isympy.1 create mode 100644 bltest/share/man/man1/ttx.1 diff --git a/bltest/bin/Activate.ps1 b/bltest/bin/Activate.ps1 new file mode 100644 index 00000000..b49d77ba --- /dev/null +++ b/bltest/bin/Activate.ps1 @@ -0,0 +1,247 @@ +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove VIRTUAL_ENV_PROMPT altogether. + if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) { + Remove-Item -Path env:VIRTUAL_ENV_PROMPT + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } + $env:VIRTUAL_ENV_PROMPT = $Prompt +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/bltest/bin/activate b/bltest/bin/activate new file mode 100644 index 00000000..968c1ce6 --- /dev/null +++ b/bltest/bin/activate @@ -0,0 +1,70 @@ +# This file must be used with "source bin/activate" *from bash* +# You cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # Call hash to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + hash -r 2> /dev/null + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + unset VIRTUAL_ENV_PROMPT + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +# on Windows, a path can contain colons and backslashes and has to be converted: +if [ "${OSTYPE:-}" = "cygwin" ] || [ "${OSTYPE:-}" = "msys" ] ; then + # transform D:\path\to\venv to /d/path/to/venv on MSYS + # and to /cygdrive/d/path/to/venv on Cygwin + export VIRTUAL_ENV=$(cygpath "/Users/jennefermaldonado/NSLS2/blop/bltest") +else + # use the path as-is + export VIRTUAL_ENV="/Users/jennefermaldonado/NSLS2/blop/bltest" +fi + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="(bltest) ${PS1:-}" + export PS1 + VIRTUAL_ENV_PROMPT="(bltest) " + export VIRTUAL_ENV_PROMPT +fi + +# Call hash to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +hash -r 2> /dev/null diff --git a/bltest/bin/activate.csh b/bltest/bin/activate.csh new file mode 100644 index 00000000..a332999c --- /dev/null +++ b/bltest/bin/activate.csh @@ -0,0 +1,27 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. + +# Created by Davide Di Blasi . +# Ported to Python 3.3 venv by Andrew Svetlov + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/Users/jennefermaldonado/NSLS2/blop/bltest" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "(bltest) $prompt" + setenv VIRTUAL_ENV_PROMPT "(bltest) " +endif + +alias pydoc python -m pydoc + +rehash diff --git a/bltest/bin/activate.fish b/bltest/bin/activate.fish new file mode 100644 index 00000000..d54f1487 --- /dev/null +++ b/bltest/bin/activate.fish @@ -0,0 +1,69 @@ +# This file must be used with "source /bin/activate.fish" *from fish* +# (https://fishshell.com/). You cannot run it directly. + +function deactivate -d "Exit virtual environment and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + set -e _OLD_FISH_PROMPT_OVERRIDE + # prevents error when using nested fish instances (Issue #93858) + if functions -q _old_fish_prompt + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + end + + set -e VIRTUAL_ENV + set -e VIRTUAL_ENV_PROMPT + if test "$argv[1]" != "nondestructive" + # Self-destruct! + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV "/Users/jennefermaldonado/NSLS2/blop/bltest" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# Unset PYTHONHOME if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # Save the current fish_prompt function as the function _old_fish_prompt. + functions -c fish_prompt _old_fish_prompt + + # With the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command. + set -l old_status $status + + # Output the venv prompt; color taken from the blue of the Python logo. + printf "%s%s%s" (set_color 4B8BBE) "(bltest) " (set_color normal) + + # Restore the return status of the previous command. + echo "exit $old_status" | . + # Output the original/"old" prompt. + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" + set -gx VIRTUAL_ENV_PROMPT "(bltest) " +end diff --git a/bltest/bin/bluesky-0MQ-proxy b/bltest/bin/bluesky-0MQ-proxy new file mode 100755 index 00000000..3ebe3710 --- /dev/null +++ b/bltest/bin/bluesky-0MQ-proxy @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from bluesky.commandline.zmq_proxy import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/convert-caffe2-to-onnx b/bltest/bin/convert-caffe2-to-onnx new file mode 100755 index 00000000..f43fa971 --- /dev/null +++ b/bltest/bin/convert-caffe2-to-onnx @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from caffe2.python.onnx.bin.conversion import caffe2_to_onnx +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(caffe2_to_onnx()) diff --git a/bltest/bin/convert-onnx-to-caffe2 b/bltest/bin/convert-onnx-to-caffe2 new file mode 100755 index 00000000..ddb0a6a4 --- /dev/null +++ b/bltest/bin/convert-onnx-to-caffe2 @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from caffe2.python.onnx.bin.conversion import onnx_to_caffe2 +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(onnx_to_caffe2()) diff --git a/bltest/bin/cpuinfo b/bltest/bin/cpuinfo new file mode 100755 index 00000000..3a519cf4 --- /dev/null +++ b/bltest/bin/cpuinfo @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from cpuinfo import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/dask b/bltest/bin/dask new file mode 100755 index 00000000..f8890612 --- /dev/null +++ b/bltest/bin/dask @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from dask.__main__ import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/f2py b/bltest/bin/f2py new file mode 100755 index 00000000..04c9379d --- /dev/null +++ b/bltest/bin/f2py @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from numpy.f2py.f2py2e import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/fonttools b/bltest/bin/fonttools new file mode 100755 index 00000000..18d10224 --- /dev/null +++ b/bltest/bin/fonttools @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from fontTools.__main__ import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/fs_rename b/bltest/bin/fs_rename new file mode 100755 index 00000000..d40042fe --- /dev/null +++ b/bltest/bin/fs_rename @@ -0,0 +1,85 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +import sys + +# This script has not been updated since the massive databroker refactor in +# v0.9.0. It needs to be revisited and tested. + +# adapted from +# https://code.activestate.com/recipes/577058/ +def query_yes_no(question, default="no"): + """Ask a yes/no question via raw_input() and return their answer. + + "question" is a string that is presented to the user. + "default" is the presumed answer if the user just hits . + It must be "yes" (the default), "no" or None (meaning + an answer is required of the user). + + The "answer" return value is one of "yes" or "no". + """ + valid = {"yes":"yes", "y":"yes", "ye":"yes", + "no":"no", "n":"no"} + if default == None: + prompt = " [y/n] " + elif default == "yes": + prompt = " [Y/n] " + elif default == "no": + prompt = " [y/N] " + else: + raise ValueError("invalid default answer: '%s'" % default) + + while 1: + choice = input(question + prompt).lower() + if default is not None and choice == '': + return default + elif choice in valid.keys(): + return valid[choice] + else: + print("Please respond with 'yes' or 'no' " + "(or 'y' or 'n').\n") + +def move_header(db, header, new_root, pre_shift, **kwargs): + resource = list(db.get_resource_uids(header)) + if pre_shift != 0: + for r in resource: + db.fs.shift_root(r, pre_shift) + for r in resource: + print("about to rename files in resource {}".format(r)) + db.fs.move_files(r, new_root, **kwargs) + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser(description='Rename all files associated with a ' + 'runstart and update filestore accordingly.\n\n' + 'This tool is not recommended for use on production data.') + parser.add_argument('header', help='runstart uid to move the files of') + parser.add_argument('server_url', help='server where mds and fs are hosted') + parser.add_argument('new_root', help='the new root to move the files to') + parser.add_argument('--preshift', type=int, + help='shift old root by this many level. ' + 'If a root has not been set in the ' + 'resource documents this must be used to ensure that old root is ' + 'non empty.', default=0) + args = parser.parse_args() + + sure = query_yes_no("You are about to rename some files, " + "are you sure you want do this?") + if sure == 'no': + sys.exit(0) + + from databroker import Broker + from databroker.core import register_builtin_handlers + from metadatastore.mds import MDS + from filestore.fs import FileStoreMoving + + mds = MDS({'host': args.server_url, 'port': 27017, + 'database': 'datastore', 'timezone': 'US/Eastern'}, + auth=False) + fs = FileStoreMoving({'host': args.server_url, 'port': 27017, + 'database': 'filestore'}) + register_builtin_handlers(fs) + db = Broker(mds, fs) + header = db[args.header] + kwargs = {'verify': False, 'remove_origin': True, + 'file_rename_hook': lambda n, N, s, d: print('{} -> {}'.format(s, d))} + move_header(db, header, args.new_root, args.preshift, **kwargs) diff --git a/bltest/bin/imageio_download_bin b/bltest/bin/imageio_download_bin new file mode 100755 index 00000000..effe539b --- /dev/null +++ b/bltest/bin/imageio_download_bin @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from imageio.__main__ import download_bin_main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(download_bin_main()) diff --git a/bltest/bin/imageio_remove_bin b/bltest/bin/imageio_remove_bin new file mode 100755 index 00000000..1002715a --- /dev/null +++ b/bltest/bin/imageio_remove_bin @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from imageio.__main__ import remove_bin_main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(remove_bin_main()) diff --git a/bltest/bin/intake b/bltest/bin/intake new file mode 100755 index 00000000..dc38194d --- /dev/null +++ b/bltest/bin/intake @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from intake.cli.client.__main__ import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/intake-server b/bltest/bin/intake-server new file mode 100755 index 00000000..ed83ba43 --- /dev/null +++ b/bltest/bin/intake-server @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from intake.cli.server.__main__ import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/isympy b/bltest/bin/isympy new file mode 100755 index 00000000..7a32a1fc --- /dev/null +++ b/bltest/bin/isympy @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from isympy import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/jsonschema b/bltest/bin/jsonschema new file mode 100755 index 00000000..e99f8159 --- /dev/null +++ b/bltest/bin/jsonschema @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from jsonschema.cli import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/lsm2bin b/bltest/bin/lsm2bin new file mode 100755 index 00000000..c7242147 --- /dev/null +++ b/bltest/bin/lsm2bin @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from tifffile.lsm2bin import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/normalizer b/bltest/bin/normalizer new file mode 100755 index 00000000..cc6fb085 --- /dev/null +++ b/bltest/bin/normalizer @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from charset_normalizer.cli import cli_detect +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(cli_detect()) diff --git a/bltest/bin/pint-convert b/bltest/bin/pint-convert new file mode 100755 index 00000000..1dc41b9b --- /dev/null +++ b/bltest/bin/pint-convert @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from pint.pint_convert import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/pip b/bltest/bin/pip new file mode 100755 index 00000000..f87906e8 --- /dev/null +++ b/bltest/bin/pip @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/pip3 b/bltest/bin/pip3 new file mode 100755 index 00000000..f87906e8 --- /dev/null +++ b/bltest/bin/pip3 @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/pip3.12 b/bltest/bin/pip3.12 new file mode 100755 index 00000000..f87906e8 --- /dev/null +++ b/bltest/bin/pip3.12 @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/pt2to3 b/bltest/bin/pt2to3 new file mode 100755 index 00000000..fcb578f6 --- /dev/null +++ b/bltest/bin/pt2to3 @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from tables.scripts.pt2to3 import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/ptdump b/bltest/bin/ptdump new file mode 100755 index 00000000..af134ff6 --- /dev/null +++ b/bltest/bin/ptdump @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from tables.scripts.ptdump import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/ptrepack b/bltest/bin/ptrepack new file mode 100755 index 00000000..6196802e --- /dev/null +++ b/bltest/bin/ptrepack @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from tables.scripts.ptrepack import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/pttree b/bltest/bin/pttree new file mode 100755 index 00000000..5094a1b9 --- /dev/null +++ b/bltest/bin/pttree @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from tables.scripts.pttree import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/pyftmerge b/bltest/bin/pyftmerge new file mode 100755 index 00000000..f3cc8257 --- /dev/null +++ b/bltest/bin/pyftmerge @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from fontTools.merge import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/pyftsubset b/bltest/bin/pyftsubset new file mode 100755 index 00000000..ab35c44d --- /dev/null +++ b/bltest/bin/pyftsubset @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from fontTools.subset import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/python b/bltest/bin/python new file mode 120000 index 00000000..11b9d885 --- /dev/null +++ b/bltest/bin/python @@ -0,0 +1 @@ +python3.12 \ No newline at end of file diff --git a/bltest/bin/python3 b/bltest/bin/python3 new file mode 120000 index 00000000..11b9d885 --- /dev/null +++ b/bltest/bin/python3 @@ -0,0 +1 @@ +python3.12 \ No newline at end of file diff --git a/bltest/bin/python3.12 b/bltest/bin/python3.12 new file mode 120000 index 00000000..f3d5d0b4 --- /dev/null +++ b/bltest/bin/python3.12 @@ -0,0 +1 @@ +/usr/local/opt/python@3.12/bin/python3.12 \ No newline at end of file diff --git a/bltest/bin/regenerate-schema b/bltest/bin/regenerate-schema new file mode 100755 index 00000000..f4690063 --- /dev/null +++ b/bltest/bin/regenerate-schema @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from event_model.documents.generate.__main__ import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/start_md_server b/bltest/bin/start_md_server new file mode 100755 index 00000000..715e42d3 --- /dev/null +++ b/bltest/bin/start_md_server @@ -0,0 +1,92 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +""" Startup script for the server.""" +import argparse +import sys +import tornado.web +import socket +import tornado.options +from databroker.headersource.mongo import MDS, MDSRO +from databroker.service.metadata import ( + RunStartHandler, RunStopHandler, + EventDescriptorHandler, + EventHandler, loop) + + +if __name__ == "__main__": + config = {} + parser = argparse.ArgumentParser() + parser.add_argument('--database', dest='database', type=str, + help='name of database to use') + parser.add_argument('--mongo-host', dest='mongohost', type=str, + help='mongodb host to connect to') + parser.add_argument('--timezone', dest='timezone', type=str, + help='Local timezone') + parser.add_argument('--mongo-port', dest='mongoport', type=int, + help='mongodb port to connect') + parser.add_argument('--service-port', dest='serviceport', type=int, + help='port to broadcast from') + parser.add_argument('--no-auth', dest='auth', action='store_false') + parser.add_argument('--auth', dest='auth', action='store_true') + parser.set_defaults(auth=False) + parser.add_argument('--mongo-user', dest='mongo_user', type=str, + help='Mongo username') + parser.add_argument('--mongo-pwd', dest='mongo_pwd', type=str, + help='Mongo password') + args = parser.parse_args() + # name of the database server will talk to. + # If db does not exist, creates one + if args.database is not None: + config['database'] = args.database + else: + raise KeyError('--database is a required arg') + # name/ip address of the machine hosting mongodb + if args.mongohost is not None: + config['mongohost'] = args.mongohost + else: + raise KeyError('--mongo-host is a required arg') + # US/Eastern for BNL + if args.timezone is not None: + config['timezone'] = args.timezone + else: + raise KeyError('--timezone is a required arg') + # port mongo uses on the mongo-host machine, 27017 by default + if args.mongoport is not None: + config['mongoport'] = args.mongoport + else: + raise KeyError('--mongo-port is a required arg') + # Port that this server will use to communicate + if args.serviceport is not None: + config['serviceport'] = args.serviceport + else: + raise KeyError('--service-port is a required arg') + if args.auth: + if args.mongo_user and args.mongo_pwd: + config['mongo_user'] = args.mongo_user + config['mongo_pwd'] = args.mongo_pwd + else: + raise KeyError('--mongo-user and --mongo-pwd required with auth') + else: + config['mongo_user'] = None + config['mongo_pwd'] = None + libconfig = dict(host=config['mongohost'], port=config['mongoport'], + timezone=config['timezone'], database=config['database'], + mongo_user=config['mongo_user'], + mongo_pwd=config['mongo_pwd']) + mdsro = MDSRO(config=libconfig, auth=args.auth) + mdsrw = MDS(config=libconfig, auth=args.auth) + + print('Connecting to mongodb...{}:{}/{}'.format(config['mongohost'], + config['mongoport'], + config['database'])) + args = sys.argv + # args.append("--log_file_prefix=/tmp/metadataservice.log") + # tornado.options.parse_command_line(args) + application = tornado.web.Application([ + (r'/run_start', RunStartHandler), (r'/run_stop', RunStopHandler), + (r'/event_descriptor', EventDescriptorHandler), + (r'/event', EventHandler) + ], mdsro=mdsro, mdsrw=mdsrw) + application.listen(config['serviceport']) + print('Service live on address {}:{}'.format(socket.gethostname(), + config['serviceport'])) + loop.start() diff --git a/bltest/bin/tiff2fsspec b/bltest/bin/tiff2fsspec new file mode 100755 index 00000000..3bca4c20 --- /dev/null +++ b/bltest/bin/tiff2fsspec @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from tifffile.tiff2fsspec import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/tiffcomment b/bltest/bin/tiffcomment new file mode 100755 index 00000000..997265d7 --- /dev/null +++ b/bltest/bin/tiffcomment @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from tifffile.tiffcomment import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/tifffile b/bltest/bin/tifffile new file mode 100755 index 00000000..63133131 --- /dev/null +++ b/bltest/bin/tifffile @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from tifffile import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/torchrun b/bltest/bin/torchrun new file mode 100755 index 00000000..74d8a1f2 --- /dev/null +++ b/bltest/bin/torchrun @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from torch.distributed.run import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/tqdm b/bltest/bin/tqdm new file mode 100755 index 00000000..19463508 --- /dev/null +++ b/bltest/bin/tqdm @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from tqdm.cli import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/bin/ttx b/bltest/bin/ttx new file mode 100755 index 00000000..6ce7a14d --- /dev/null +++ b/bltest/bin/ttx @@ -0,0 +1,8 @@ +#!/Users/jennefermaldonado/NSLS2/blop/bltest/bin/python3.12 +# -*- coding: utf-8 -*- +import re +import sys +from fontTools.ttx import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/bltest/include/b2nd.h b/bltest/include/b2nd.h new file mode 100644 index 00000000..d3a5f071 --- /dev/null +++ b/bltest/include/b2nd.h @@ -0,0 +1,581 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (c) 2021 Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/** @file b2nd.h + * @brief Blosc2 NDim header file. + * + * This file contains Blosc2 NDim public API and the structures needed to use it. + * @author Blosc Development Team + */ + +#ifndef BLOSC_B2ND_H +#define BLOSC_B2ND_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "blosc2/blosc2-export.h" +#ifdef __cplusplus +} +#endif + +#include "blosc2.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +/* The version for metalayer format; starts from 0 and it must not exceed 127 */ +#define B2ND_METALAYER_VERSION 0 + +/* The maximum number of dimensions for b2nd arrays */ +#define B2ND_MAX_DIM 8 + +/* The maximum number of metalayers for b2nd arrays */ +#define B2ND_MAX_METALAYERS (BLOSC2_MAX_METALAYERS - 1) + +/* NumPy dtype format + * https://numpy.org/doc/stable/reference/arrays.dtypes.html#arrays-dtypes-constructing + */ +#define DTYPE_NUMPY_FORMAT 0 + +/* The default data type */ +#define B2ND_DEFAULT_DTYPE "|u1" +/* The default data format */ +#define B2ND_DEFAULT_DTYPE_FORMAT DTYPE_NUMPY_FORMAT + +/** + * @brief An *optional* cache for a single block. + * + * When a chunk is needed, it is copied into this cache. In this way, if the same chunk is needed + * again afterwards, it is not necessary to recover it because it is already in the cache. + */ +struct chunk_cache_s { + uint8_t *data; + //!< The chunk data. + int64_t nchunk; + //!< The chunk number in cache. If @p nchunk equals to -1, it means that the cache is empty. +}; + +/** + * @brief General parameters needed for the creation of a b2nd array. + */ +typedef struct b2nd_context_s b2nd_context_t; /* opaque type */ + +/** + * @brief A multidimensional array of data that can be compressed. + */ +typedef struct { + blosc2_schunk *sc; + //!< Pointer to a Blosc super-chunk + int64_t shape[B2ND_MAX_DIM]; + //!< Shape of original data. + int32_t chunkshape[B2ND_MAX_DIM]; + //!< Shape of each chunk. + int64_t extshape[B2ND_MAX_DIM]; + //!< Shape of padded data. + int32_t blockshape[B2ND_MAX_DIM]; + //!< Shape of each block. + int64_t extchunkshape[B2ND_MAX_DIM]; + //!< Shape of padded chunk. + int64_t nitems; + //!< Number of items in original data. + int32_t chunknitems; + //!< Number of items in each chunk. + int64_t extnitems; + //!< Number of items in padded data. + int32_t blocknitems; + //!< Number of items in each block. + int64_t extchunknitems; + //!< Number of items in a padded chunk. + int8_t ndim; + //!< Data dimensions. + struct chunk_cache_s chunk_cache; + //!< A partition cache. + int64_t item_array_strides[B2ND_MAX_DIM]; + //!< Item - shape strides. + int64_t item_chunk_strides[B2ND_MAX_DIM]; + //!< Item - shape strides. + int64_t item_extchunk_strides[B2ND_MAX_DIM]; + //!< Item - shape strides. + int64_t item_block_strides[B2ND_MAX_DIM]; + //!< Item - shape strides. + int64_t block_chunk_strides[B2ND_MAX_DIM]; + //!< Item - shape strides. + int64_t chunk_array_strides[B2ND_MAX_DIM]; + //!< Item - shape strides. + char *dtype; + //!< Data type. Different formats can be supported (see dtype_format). + int8_t dtype_format; + //!< The format of the data type. Default is DTYPE_NUMPY_FORMAT. +} b2nd_array_t; + + +/** + * @brief Create b2nd params. + * + * @param b2_storage The Blosc2 storage params. + * @param ndim The dimensions. + * @param shape The shape. + * @param chunkshape The chunk shape. + * @param blockshape The block shape. + * @param dtype The data type expressed as a string version. + * @param dtype_format The data type format; DTYPE_NUMPY_FORMAT should be chosen for NumPy compatibility. + * @param metalayers The memory pointer to the list of the metalayers desired. + * @param nmetalayers The number of metalayers. + * + * @return A pointer to the new b2nd params. NULL is returned if this fails. + * + * @note The pointer returned must be freed when not used anymore with #b2nd_free_ctx. + * + */ +BLOSC_EXPORT b2nd_context_t * +b2nd_create_ctx(const blosc2_storage *b2_storage, int8_t ndim, const int64_t *shape, const int32_t *chunkshape, + const int32_t *blockshape, const char *dtype, int8_t dtype_format, const blosc2_metalayer *metalayers, + int32_t nmetalayers); + + +/** + * @brief Free the resources associated with b2nd_context_t. + * + * @param ctx The b2nd context to free. + * + * @return An error code. + * + * @note This is safe in the sense that it will not free the schunk pointer in internal cparams. + * + */ +BLOSC_EXPORT int b2nd_free_ctx(b2nd_context_t *ctx); + + +/** + * @brief Create an uninitialized array. + * + * @param ctx The b2nd context for the new array. + * @param array The memory pointer where the array will be created. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_uninit(b2nd_context_t *ctx, b2nd_array_t **array); + + +/** + * @brief Create an empty array. + * + * @param ctx The b2nd context for the new array. + * @param array The memory pointer where the array will be created. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_empty(b2nd_context_t *ctx, b2nd_array_t **array); + + +/** + * Create an array, with zero being used as the default value for + * uninitialized portions of the array. + * + * @param ctx The b2nd context for the new array. + * @param array The memory pointer where the array will be created. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_zeros(b2nd_context_t *ctx, b2nd_array_t **array); + + +/** + * Create an array, with NaN being used as the default value for + * uninitialized portions of the array. Should only be used with type sizes + * of either 4 or 8. Other sizes generate an error. + * + * @param ctx The b2nd context for the new array. + * @param array The memory pointer where the array will be created. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_nans(b2nd_context_t *ctx, b2nd_array_t **array); + + +/** + * Create an array, with @p fill_value being used as the default value for + * uninitialized portions of the array. + * + * @param ctx The b2nd context for the new array. + * @param array The memory pointer where the array will be created. + * @param fill_value Default value for uninitialized portions of the array. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_full(b2nd_context_t *ctx, b2nd_array_t **array, const void *fill_value); + +/** + * @brief Free an array. + * + * @param array The array. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_free(b2nd_array_t *array); + +/** + * @brief Create a b2nd array from a super-chunk. It can only be used if the array + * is backed by a blosc super-chunk. + * + * @param schunk The blosc super-chunk where the b2nd array is stored. + * @param array The memory pointer where the array will be created. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_from_schunk(blosc2_schunk *schunk, b2nd_array_t **array); + +/** + * Create a serialized super-chunk from a b2nd array. + * + * @param array The b2nd array to be serialized. + * @param cframe The pointer of the buffer where the in-memory array will be copied. + * @param cframe_len The length of the in-memory array buffer. + * @param needs_free Whether the buffer should be freed or not. + * + * @return An error code + */ +BLOSC_EXPORT int b2nd_to_cframe(const b2nd_array_t *array, uint8_t **cframe, + int64_t *cframe_len, bool *needs_free); + +/** + * @brief Create a b2nd array from a serialized super-chunk. + * + * @param cframe The buffer of the in-memory array. + * @param cframe_len The size (in bytes) of the in-memory array. + * @param copy Whether b2nd should make a copy of the cframe data or not. The copy will be made to an internal sparse frame. + * @param array The memory pointer where the array will be created. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_from_cframe(uint8_t *cframe, int64_t cframe_len, bool copy, b2nd_array_t **array); + +/** + * @brief Open a b2nd array from a file. + * + * @param urlpath The path of the b2nd array on disk. + * @param array The memory pointer where the array info will be stored. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_open(const char *urlpath, b2nd_array_t **array); + +/** + * @brief Open a b2nd array from a file using an offset. + * + * @param urlpath The path of the b2nd array on disk. + * @param array The memory pointer where the array info will be stored. + * @param offset The offset in the file where the b2nd array frame starts. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_open_offset(const char *urlpath, b2nd_array_t **array, int64_t offset); + +/** + * @brief Save b2nd array into a specific urlpath. + * + * @param array The array to be saved. + * @param urlpath The urlpath where the array will be stored. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_save(const b2nd_array_t *array, char *urlpath); + +/** + * @brief Create a b2nd array from a C buffer. + * + * @param ctx The b2nd context for the new array. + * @param array The memory pointer where the array will be created. + * @param buffer The buffer where source data is stored. + * @param buffersize The size (in bytes) of the buffer. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_from_cbuffer(b2nd_context_t *ctx, b2nd_array_t **array, const void *buffer, int64_t buffersize); + +/** + * @brief Extract the data from a b2nd array into a C buffer. + * + * @param array The b2nd array. + * @param buffer The buffer where the data will be stored. + * @param buffersize Size (in bytes) of the buffer. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_to_cbuffer(const b2nd_array_t *array, void *buffer, int64_t buffersize); + +/** + * @brief Get a slice from an array and store it into a new array. + * + * @param ctx The b2nd context for the new array. + * @param array The memory pointer where the array will be created. + * @param src The array from which the slice will be extracted + * @param start The coordinates where the slice will begin. + * @param stop The coordinates where the slice will end. + * + * @return An error code. + * + * @note The ndim and shape from ctx will be overwritten by the src and stop-start respectively. + * + */ +BLOSC_EXPORT int b2nd_get_slice(b2nd_context_t *ctx, b2nd_array_t **array, const b2nd_array_t *src, + const int64_t *start, const int64_t *stop); + +/** + * @brief Squeeze a b2nd array + * + * This function remove selected single-dimensional entries from the shape of a + b2nd array. + * + * @param array The b2nd array. + * @param index Indexes of the single-dimensional entries to remove. + * + * @return An error code + */ +BLOSC_EXPORT int b2nd_squeeze_index(b2nd_array_t *array, const bool *index); + +/** + * @brief Squeeze a b2nd array + * + * This function remove single-dimensional entries from the shape of a b2nd array. + * + * @param array The b2nd array. + * + * @return An error code + */ +BLOSC_EXPORT int b2nd_squeeze(b2nd_array_t *array); + +/** + * @brief Get a slice from an array and store it into a C buffer. + * + * @param array The array from which the slice will be extracted. + * @param start The coordinates where the slice will begin. + * @param stop The coordinates where the slice will end. + * @param buffershape The shape of the buffer. + * @param buffer The buffer where the data will be stored. + * @param buffersize The size (in bytes) of the buffer. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_get_slice_cbuffer(const b2nd_array_t *array, const int64_t *start, const int64_t *stop, + void *buffer, const int64_t *buffershape, int64_t buffersize); + +/** + * @brief Set a slice in a b2nd array using a C buffer. + * + * @param buffer The buffer where the slice data is. + * @param buffershape The shape of the buffer. + * @param buffersize The size (in bytes) of the buffer. + * @param start The coordinates where the slice will begin. + * @param stop The coordinates where the slice will end. + * @param array The b2nd array where the slice will be set + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_set_slice_cbuffer(const void *buffer, const int64_t *buffershape, int64_t buffersize, + const int64_t *start, const int64_t *stop, b2nd_array_t *array); + +/** + * @brief Make a copy of the array data. The copy is done into a new b2nd array. + * + * @param ctx The b2nd context for the new array. + * @param src The array from which data is copied. + * @param array The memory pointer where the array will be created. + * + * @return An error code + * + * @note The ndim and shape in ctx will be overwritten by the src ctx. + * + */ +BLOSC_EXPORT int b2nd_copy(b2nd_context_t *ctx, const b2nd_array_t *src, b2nd_array_t **array); + +/** + * @brief Print metalayer parameters. + * + * @param array The array where the metalayer is stored. + * + * @return An error code + */ +BLOSC_EXPORT int b2nd_print_meta(const b2nd_array_t *array); + +/** + * @brief Resize the shape of an array + * + * @param array The array to be resized. + * @param new_shape The new shape from the array. + * @param start The position in which the array will be extended or shrunk. + * + * @return An error code + */ +BLOSC_EXPORT int b2nd_resize(b2nd_array_t *array, const int64_t *new_shape, const int64_t *start); + + +/** + * @brief Insert given buffer in an array extending the given axis. + * + * @param array The array to insert the data in. + * @param buffer The buffer data to be inserted. + * @param buffersize The size (in bytes) of the buffer. + * @param axis The axis that will be extended. + * @param insert_start The position inside the axis to start inserting the data. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_insert(b2nd_array_t *array, const void *buffer, int64_t buffersize, + int8_t axis, int64_t insert_start); + +/** + * Append a buffer at the end of a b2nd array. + * + * @param array The array to append the data in. + * @param buffer The buffer data to be appended. + * @param buffersize Size (in bytes) of the buffer. + * @param axis The axis that will be extended to append the data. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_append(b2nd_array_t *array, const void *buffer, int64_t buffersize, + int8_t axis); + +/** + * @brief Delete shrinking the given axis delete_len items. + * + * @param array The array to shrink. + * @param axis The axis to shrink. + * @param delete_start The start position from the axis to start deleting chunks. + * @param delete_len The number of items to delete to the array->shape[axis]. + * The newshape[axis] will be the old array->shape[axis] - delete_len + * + * @return An error code. + * + * @note See also b2nd_resize + */ +BLOSC_EXPORT int b2nd_delete(b2nd_array_t *array, int8_t axis, + int64_t delete_start, int64_t delete_len); + + +// Indexing section + +/** + * @brief Get an element selection along each dimension of an array independently. + * + * @param array The array to get the data from. + * @param selection The elements along each dimension. + * @param selection_size The size of the selection along each dimension. + * @param buffer The buffer for getting the data. + * @param buffershape The shape of the buffer. + * @param buffersize The buffer size (in bytes). + * + * @return An error code. + * + * @note See also b2nd_set_orthogonal_selection. + */ +BLOSC_EXPORT int b2nd_get_orthogonal_selection(const b2nd_array_t *array, int64_t **selection, + int64_t *selection_size, void *buffer, + int64_t *buffershape, int64_t buffersize); + +/** + * @brief Set an element selection along each dimension of an array independently. + * + * @param array The array to set the data to. + * @param selection The elements along each dimension. + * @param selection_size The size of the selection along each dimension. + * @param buffer The buffer with the data for setting. + * @param buffershape The shape of the buffer. + * @param buffersize The buffer size (in bytes). + * + * @return An error code. + * + * @note See also b2nd_get_orthogonal_selection. + */ +BLOSC_EXPORT int b2nd_set_orthogonal_selection(b2nd_array_t *array, int64_t **selection, + int64_t *selection_size, const void *buffer, + int64_t *buffershape, int64_t buffersize); + + +/** + * @brief Create the metainfo for the b2nd metalayer. + * + * @param ndim The number of dimensions in the array. + * @param shape The shape of the array. + * @param chunkshape The shape of the chunks in the array. + * @param blockshape The shape of the blocks in the array. + * @param dtype A string representation of the data type of the array. + * @param dtype_format The format of the dtype representation. 0 means NumPy. + * @param smeta The msgpack buffer (output). + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_serialize_meta(int8_t ndim, const int64_t *shape, const int32_t *chunkshape, + const int32_t *blockshape, const char *dtype, + int8_t dtype_format, uint8_t **smeta); + +/** + * @brief Read the metainfo in the b2nd metalayer. + * + * @param smeta The msgpack buffer (input). + * @param smeta_len The length of the smeta buffer (input). + * @param ndim The number of dimensions in the array (output). + * @param shape The shape of the array (output). + * @param chunkshape The shape of the chunks in the array (output). + * @param blockshape The shape of the blocks in the array (output). + * @param dtype A string representation of the data type of the array (output). + * @param dtype_format The format of the dtype representation (output). 0 means NumPy (the default). + * + * @note This function is inlined and available even when not linking with libblosc2. + * + * @return An error code. + */ +BLOSC_EXPORT int b2nd_deserialize_meta(const uint8_t *smeta, int32_t smeta_len, int8_t *ndim, int64_t *shape, + int32_t *chunkshape, int32_t *blockshape, char **dtype, int8_t *dtype_format); + +// Utilities for C buffers representing multidimensional arrays + +/** + * @brief Copy a slice of a source array into another array. The arrays have + * the same number of dimensions (though their shapes may differ), the same + * item size, and they are stored as C buffers with contiguous data (any + * padding is considered part of the array). + * + * @param ndim The number of dimensions in both arrays. + * @param itemsize The size of the individual data item in both arrays. + * @param src The buffer for getting the data from the source array. + * @param src_pad_shape The shape of the source array, including padding. + * @param src_start The source coordinates where the slice will begin. + * @param src_stop The source coordinates where the slice will end. + * @param dst The buffer for setting the data into the destination array. + * @param dst_pad_shape The shape of the destination array, including padding. + * @param dst_start The destination coordinates where the slice will be placed. + * + * @return An error code. + * + * @note Please make sure that slice boundaries fit within the source and + * destination arrays before using this function, as it does not perform these + * checks itself. + */ +BLOSC_EXPORT int b2nd_copy_buffer(int8_t ndim, + uint8_t itemsize, + const void *src, const int64_t *src_pad_shape, + const int64_t *src_start, const int64_t *src_stop, + void *dst, const int64_t *dst_pad_shape, + const int64_t *dst_start); + + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_B2ND_H */ diff --git a/bltest/include/blosc2.h b/bltest/include/blosc2.h new file mode 100644 index 00000000..f5f92efa --- /dev/null +++ b/bltest/include/blosc2.h @@ -0,0 +1,2608 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (c) 2021 Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +/********************************************************************* + @file blosc2.h + @brief Blosc2 header file. + + This file contains Blosc2 public API and the structures needed to use it. + @author Blosc Development Team +**********************************************************************/ + +#ifndef BLOSC_BLOSC2_H +#define BLOSC_BLOSC2_H + +#include "blosc2/blosc2-export.h" +#include "blosc2/blosc2-common.h" +#include "blosc2/blosc2-stdio.h" + +#if defined(_WIN32) && !defined(__MINGW32__) +#include +#include +#include +#define getpid _getpid +#endif + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// For compatibility with the Blosc 1.x series +#ifdef BLOSC1_COMPAT + // Blosc2 symbols that should be accessible from Blosc 1.x API + #define BLOSC_VERSION_MAJOR BLOSC2_VERSION_MAJOR + #define BLOSC_VERSION_MINOR BLOSC2_VERSION_MINOR + #define BLOSC_VERSION_RELEASE BLOSC2_VERSION_RELEASE + #define BLOSC_VERSION_STRING BLOSC2_VERSION_STRING + #define BLOSC_VERSION_DATE BLOSC2_VERSION_DATE + #define BLOSC_MAX_OVERHEAD BLOSC2_MAX_OVERHEAD + #define BLOSC_MAX_BUFFERSIZE BLOSC2_MAX_BUFFERSIZE + + // API that changed to blosc1_ prefix + #define blosc_compress blosc1_compress + #define blosc_decompress blosc1_decompress + #define blosc_getitem blosc1_getitem + #define blosc_get_compressor blosc1_get_compressor + #define blosc_set_compressor blosc1_set_compressor + #define blosc_cbuffer_sizes blosc1_cbuffer_sizes + #define blosc_cbuffer_validate blosc1_cbuffer_validate + #define blosc_cbuffer_metainfo blosc1_cbuffer_metainfo + #define blosc_get_blocksize blosc1_get_blocksize + #define blosc_set_blocksize blosc1_set_blocksize + #define blosc_set_splitmode blosc1_set_splitmode + + // API that changed to blosc2_ prefix + #define blosc_init blosc2_init + #define blosc_destroy blosc2_destroy + #define blosc_free_resources blosc2_free_resources + #define blosc_get_nthreads blosc2_get_nthreads + #define blosc_set_nthreads blosc2_set_nthreads + #define blosc_compcode_to_compname blosc2_compcode_to_compname + #define blosc_compname_to_compcode blosc2_compname_to_compcode + #define blosc_list_compressors blosc2_list_compressors + #define blosc_get_version_string blosc2_get_version_string + #define blosc_get_complib_info blosc2_get_complib_info + #define blosc_cbuffer_versions blosc2_cbuffer_versions + #define blosc_cbuffer_complib blosc2_cbuffer_complib +#endif + + +/* Version numbers */ +#define BLOSC2_VERSION_MAJOR 2 /* for major interface/format changes */ +#define BLOSC2_VERSION_MINOR 15 /* for minor interface/format changes */ +#define BLOSC2_VERSION_RELEASE 2 /* for tweaks, bug-fixes, or development */ + +#define BLOSC2_VERSION_STRING "2.15.2.dev" /* string version. Sync with above! */ +#define BLOSC2_VERSION_DATE "$Date:: 2024-07-30 #$" /* date version year-month-day */ + + +/* The maximum number of dimensions for Blosc2 NDim arrays */ +#define BLOSC2_MAX_DIM 8 + + +/* Tracing macros */ +#define BLOSC_TRACE_ERROR(msg, ...) BLOSC_TRACE(error, msg, ##__VA_ARGS__) +#define BLOSC_TRACE_WARNING(msg, ...) BLOSC_TRACE(warning, msg, ##__VA_ARGS__) +#define BLOSC_TRACE_INFO(msg, ...) BLOSC_TRACE(info, msg, ##__VA_ARGS__) +#define BLOSC_TRACE(cat, msg, ...) \ + do { \ + const char *__e = getenv("BLOSC_TRACE"); \ + if (!__e) { break; } \ + fprintf(stderr, "[%s] - " msg " (%s:%d)\n", #cat, ##__VA_ARGS__, __FILE__, __LINE__); \ + } while(0) + +#define BLOSC_ERROR_NULL(pointer, rc) \ + do { \ + if ((pointer) == NULL) { \ + BLOSC_TRACE_ERROR("Pointer is null"); \ + return (rc); \ + } \ + } while (0) +#define BLOSC_ERROR(rc) \ + do { \ + int rc_ = (rc); \ + if (rc_ < BLOSC2_ERROR_SUCCESS) { \ + char *error_msg = print_error(rc_); \ + BLOSC_TRACE_ERROR("%s", error_msg); \ + return rc_; \ + } \ + } while (0) + +#define BLOSC_INFO(msg, ...) \ + do { \ + const char *__e = getenv("BLOSC_INFO"); \ + if (!__e) { break; } \ + fprintf(stderr, "[INFO] - " msg "\n", ##__VA_ARGS__); \ + } while(0) + + +/* The VERSION_FORMAT symbols below should be just 1-byte long */ +enum { + /* Blosc format version, starting at 1 + 1 -> Blosc pre-1.0 + 2 -> Blosc 1.x stable series + 3 -> Blosc 2-alpha.x series + 4 -> Blosc 2.x beta.1 series + 5 -> Blosc 2.x stable series + */ + BLOSC1_VERSION_FORMAT_PRE1 = 1, + BLOSC1_VERSION_FORMAT = 2, + BLOSC2_VERSION_FORMAT_ALPHA = 3, + BLOSC2_VERSION_FORMAT_BETA1 = 4, + BLOSC2_VERSION_FORMAT_STABLE = 5, + BLOSC2_VERSION_FORMAT = BLOSC2_VERSION_FORMAT_STABLE, +}; + + +/* The FRAME_FORMAT_VERSION symbols below should be just 4-bit long */ +enum { + /* Blosc format version + * 1 -> First version (introduced in beta.2) + * 2 -> Second version (introduced in rc.1) + * + */ + BLOSC2_VERSION_FRAME_FORMAT_BETA2 = 1, // for 2.0.0-beta2 and after + BLOSC2_VERSION_FRAME_FORMAT_RC1 = 2, // for 2.0.0-rc1 and after + BLOSC2_VERSION_FRAME_FORMAT = BLOSC2_VERSION_FRAME_FORMAT_RC1, +}; + + +//!< Struct for storing data from instrumentation of codecs +// This can be flexible because it is typically used mainly for development +typedef struct { + float cratio; + float cspeed; + float filter_speed; + //float memory; + //float power; + uint8_t flags[4]; +} blosc2_instr; + + +enum { +#ifndef BLOSC_H + BLOSC_MIN_HEADER_LENGTH = 16, + //!< Minimum header length (Blosc1) +#endif // BLOSC_H + BLOSC_EXTENDED_HEADER_LENGTH = 32, + //!< Extended header length (Blosc2, see README_HEADER) + BLOSC2_MAX_OVERHEAD = BLOSC_EXTENDED_HEADER_LENGTH, + //!< The maximum overhead during compression in bytes. This equals + //!< to @ref BLOSC_EXTENDED_HEADER_LENGTH now, but can be higher in future + //!< implementations. + BLOSC2_MAX_BUFFERSIZE = (INT_MAX - BLOSC2_MAX_OVERHEAD), + //!< Maximum source buffer size to be compressed +#ifndef BLOSC_H + BLOSC_MAX_TYPESIZE = UINT8_MAX, + //!< Maximum typesize before considering source buffer as a stream of bytes. + //!< Cannot be larger than 255. +#endif // BLOSC_H + BLOSC_MIN_BUFFERSIZE = 32, + //!< Minimum buffer size to be compressed. +}; + +enum { + BLOSC2_DEFINED_TUNER_START = 0, + BLOSC2_DEFINED_TUNER_STOP = 31, + //!< Blosc-defined tuners must be between 0 - 31. + BLOSC2_GLOBAL_REGISTERED_TUNER_START = 32, + BLOSC2_GLOBAL_REGISTERED_TUNER_STOP = 159, + //!< Blosc-registered tuners must be between 31 - 159. + BLOSC2_GLOBAL_REGISTERED_TUNERS = 0, + //!< Number of Blosc-registered tuners at the moment. + BLOSC2_USER_REGISTERED_TUNER_START = 160, + BLOSC2_USER_REGISTERED_TUNER_STOP = 255, + //!< User-defined tuners must be between 160 - 255. +}; + +/** + * @brief Codes for the different tuners shipped with Blosc + */ +enum { + BLOSC_STUNE = 0, + BLOSC_LAST_TUNER = 1, + //!< Determine the last tuner defined by Blosc. + BLOSC_LAST_REGISTERED_TUNE = BLOSC2_GLOBAL_REGISTERED_TUNER_START + BLOSC2_GLOBAL_REGISTERED_TUNERS - 1, + //!< Determine the last registered tuner. It is used to check if a tuner is registered or not. +}; + +enum { + BLOSC2_DEFINED_FILTERS_START = 0, + BLOSC2_DEFINED_FILTERS_STOP = 31, + //!< Blosc-defined filters must be between 0 - 31. + BLOSC2_GLOBAL_REGISTERED_FILTERS_START = 32, + BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP = 159, + //!< Blosc-registered filters must be between 32 - 159. + BLOSC2_GLOBAL_REGISTERED_FILTERS = 5, + //!< Number of Blosc-registered filters at the moment. + BLOSC2_USER_REGISTERED_FILTERS_START = 160, + BLOSC2_USER_REGISTERED_FILTERS_STOP = 255, + //!< User-defined filters must be between 128 - 255. + BLOSC2_MAX_FILTERS = 6, + //!< Maximum number of filters in the filter pipeline + BLOSC2_MAX_UDFILTERS = 16, + //!< Maximum number of filters that a user can register. +}; + + +/** + * @brief Codes for filters. + * + * @sa #blosc1_compress + */ +enum { +#ifndef BLOSC_H + BLOSC_NOSHUFFLE = 0, + //!< No shuffle (for compatibility with Blosc1). + BLOSC_NOFILTER = 0, + //!< No filter. + BLOSC_SHUFFLE = 1, + //!< Byte-wise shuffle. `filters_meta` does not have any effect here. + BLOSC_BITSHUFFLE = 2, + //!< Bit-wise shuffle. `filters_meta` does not have any effect here. +#endif // BLOSC_H + BLOSC_DELTA = 3, + //!< Delta filter. `filters_meta` does not have any effect here. + BLOSC_TRUNC_PREC = 4, + //!< Truncate mantissa precision. + //!< Positive values in `filters_meta` will keep bits; negative values will zero bits. + BLOSC_LAST_FILTER = 5, + //!< sentinel + BLOSC_LAST_REGISTERED_FILTER = BLOSC2_GLOBAL_REGISTERED_FILTERS_START + BLOSC2_GLOBAL_REGISTERED_FILTERS - 1, + //!< Determine the last registered filter. It is used to check if a filter is registered or not. +}; + +/** + * @brief Codes for internal flags (see blosc1_cbuffer_metainfo) + */ +enum { +#ifndef BLOSC_H + BLOSC_DOSHUFFLE = 0x1, //!< byte-wise shuffle + BLOSC_MEMCPYED = 0x2, //!< plain copy + BLOSC_DOBITSHUFFLE = 0x4, //!< bit-wise shuffle +#endif // BLOSC_H + BLOSC_DODELTA = 0x8, //!< delta coding +}; + +/** + * @brief Codes for new internal flags in Blosc2 + */ +enum { + BLOSC2_USEDICT = 0x1, //!< use dictionaries with codec + BLOSC2_BIGENDIAN = 0x2, //!< data is in big-endian ordering + BLOSC2_INSTR_CODEC = 0x80, //!< codec is instrumented (mainly for development) +}; + +/** + * @brief Values for different Blosc2 capabilities + */ +enum { + BLOSC2_MAXDICTSIZE = 128 * 1024, //!< maximum size for compression dicts + BLOSC2_MAXBLOCKSIZE = 536866816 //!< maximum size for blocks +}; + + +enum { + BLOSC2_DEFINED_CODECS_START = 0, + BLOSC2_DEFINED_CODECS_STOP = 31, + //!< Blosc-defined codecs must be between 0 - 31. + BLOSC2_GLOBAL_REGISTERED_CODECS_START = 32, + BLOSC2_GLOBAL_REGISTERED_CODECS_STOP = 159, + //!< Blosc-registered codecs must be between 31 - 159. + BLOSC2_GLOBAL_REGISTERED_CODECS = 5, + //!< Number of Blosc-registered codecs at the moment. + BLOSC2_USER_REGISTERED_CODECS_START = 160, + BLOSC2_USER_REGISTERED_CODECS_STOP = 255, + //!< User-defined codecs must be between 160 - 255. +}; + +/** + * @brief Codes for the different compressors shipped with Blosc + */ +enum { +#ifndef BLOSC_H + BLOSC_BLOSCLZ = 0, + BLOSC_LZ4 = 1, + BLOSC_LZ4HC = 2, + BLOSC_ZLIB = 4, + BLOSC_ZSTD = 5, +#endif // BLOSC_H + BLOSC_LAST_CODEC = 6, + //!< Determine the last codec defined by Blosc. + BLOSC_LAST_REGISTERED_CODEC = BLOSC2_GLOBAL_REGISTERED_CODECS_START + BLOSC2_GLOBAL_REGISTERED_CODECS - 1, + //!< Determine the last registered codec. It is used to check if a codec is registered or not. +}; + + +// Names for the different compressors shipped with Blosc + +#ifndef BLOSC_H +#define BLOSC_BLOSCLZ_COMPNAME "blosclz" +#define BLOSC_LZ4_COMPNAME "lz4" +#define BLOSC_LZ4HC_COMPNAME "lz4hc" +#define BLOSC_ZLIB_COMPNAME "zlib" +#define BLOSC_ZSTD_COMPNAME "zstd" +#endif // BLOSC_H + +/** + * @brief Codes for compression libraries shipped with Blosc (code must be < 8) + */ +enum { +#ifndef BLOSC_H + BLOSC_BLOSCLZ_LIB = 0, + BLOSC_LZ4_LIB = 1, + BLOSC_ZLIB_LIB = 3, + BLOSC_ZSTD_LIB = 4, +#endif // BLOSC_H + BLOSC_UDCODEC_LIB = 6, + BLOSC_SCHUNK_LIB = 7, //!< compressor library in super-chunk header +}; + +/** + * @brief Names for the different compression libraries shipped with Blosc + */ +#ifndef BLOSC_H +#define BLOSC_BLOSCLZ_LIBNAME "BloscLZ" +#define BLOSC_LZ4_LIBNAME "LZ4" +#define BLOSC_ZLIB_LIBNAME "Zlib" +#define BLOSC_ZSTD_LIBNAME "Zstd" +#endif // BLOSC_H + +/** + * @brief The codes for compressor formats shipped with Blosc + */ +enum { +#ifndef BLOSC_H + BLOSC_BLOSCLZ_FORMAT = BLOSC_BLOSCLZ_LIB, + BLOSC_LZ4_FORMAT = BLOSC_LZ4_LIB, + //!< LZ4HC and LZ4 share the same format + BLOSC_LZ4HC_FORMAT = BLOSC_LZ4_LIB, + BLOSC_ZLIB_FORMAT = BLOSC_ZLIB_LIB, + BLOSC_ZSTD_FORMAT = BLOSC_ZSTD_LIB, +#endif // BLOSC_H + BLOSC_UDCODEC_FORMAT = BLOSC_UDCODEC_LIB, +}; + +/** + * @brief The version formats for compressors shipped with Blosc. + * All versions here starts at 1 + */ +enum { +#ifndef BLOSC_H + BLOSC_BLOSCLZ_VERSION_FORMAT = 1, + BLOSC_LZ4_VERSION_FORMAT = 1, + BLOSC_LZ4HC_VERSION_FORMAT = 1, /* LZ4HC and LZ4 share the same format */ + BLOSC_ZLIB_VERSION_FORMAT = 1, + BLOSC_ZSTD_VERSION_FORMAT = 1, +#endif // BLOSC_H + BLOSC_UDCODEC_VERSION_FORMAT = 1, +}; + +/** + * @brief Split mode for blocks. + * NEVER and ALWAYS are for experimenting with compression ratio. + * AUTO for nearly optimal behaviour (based on heuristics). + * FORWARD_COMPAT provides best forward compatibility (default). + */ +#ifndef BLOSC_H +enum { + BLOSC_ALWAYS_SPLIT = 1, + BLOSC_NEVER_SPLIT = 2, + BLOSC_AUTO_SPLIT = 3, + BLOSC_FORWARD_COMPAT_SPLIT = 4, +}; +#endif // BLOSC_H + +/** + * @brief Offsets for fields in Blosc2 chunk header. + */ +enum { + BLOSC2_CHUNK_VERSION = 0x0, //!< the version for the chunk format + BLOSC2_CHUNK_VERSIONLZ = 0x1, //!< the version for the format of internal codec + BLOSC2_CHUNK_FLAGS = 0x2, //!< flags and codec info + BLOSC2_CHUNK_TYPESIZE = 0x3, //!< (uint8) the number of bytes of the atomic type + BLOSC2_CHUNK_NBYTES = 0x4, //!< (int32) uncompressed size of the buffer (this header is not included) + BLOSC2_CHUNK_BLOCKSIZE = 0x8, //!< (int32) size of internal blocks + BLOSC2_CHUNK_CBYTES = 0xc, //!< (int32) compressed size of the buffer (including this header) + BLOSC2_CHUNK_FILTER_CODES = 0x10, //!< the codecs for the filter pipeline (1 byte per code) + BLOSC2_CHUNK_FILTER_META = 0x18, //!< meta info for the filter pipeline (1 byte per code) + BLOSC2_CHUNK_BLOSC2_FLAGS = 0x1F, //!< flags specific for Blosc2 functionality +}; + +/** + * @brief Run lengths for special values for chunks/frames + */ +enum { + BLOSC2_NO_SPECIAL = 0x0, //!< no special value + BLOSC2_SPECIAL_ZERO = 0x1, //!< zero special value + BLOSC2_SPECIAL_NAN = 0x2, //!< NaN special value + BLOSC2_SPECIAL_VALUE = 0x3, //!< generic special value + BLOSC2_SPECIAL_UNINIT = 0x4, //!< non initialized values + BLOSC2_SPECIAL_LASTID = 0x4, //!< last valid ID for special value (update this adequately) + BLOSC2_SPECIAL_MASK = 0x7 //!< special value mask (prev IDs cannot be larger than this) +}; + +/** + * @brief Error codes + * Each time an error code is added here, its corresponding message error should be added in + * print_error() + */ +enum { + BLOSC2_ERROR_SUCCESS = 0, //=0). In case the compressor + * is not recognized, or there is not support for it in this build, + * it returns a -1. + */ +BLOSC_EXPORT int blosc1_set_compressor(const char* compname); + + +/** + * @brief Select the delta coding filter to be used. + * + * @param dodelta A value >0 will activate the delta filter. + * If 0, it will be de-activated + * + * This call should always succeed. + */ +BLOSC_EXPORT void blosc2_set_delta(int dodelta); + + +/** + * @brief Get the compressor name associated with the compressor code. + * + * @param compcode The code identifying the compressor + * @param compname The pointer to a string where the compressor name will be put. + * + * @return The compressor code. If the compressor code is not recognized, + * or there is not support for it in this build, -1 is returned. + */ +BLOSC_EXPORT int blosc2_compcode_to_compname(int compcode, const char** compname); + + +/** + * @brief Get the compressor code associated with the compressor name. + * + * @param compname The string containing the compressor name. + * + * @return The compressor code. If the compressor name is not recognized, + * or there is not support for it in this build, -1 is returned instead. + */ +BLOSC_EXPORT int blosc2_compname_to_compcode(const char* compname); + + +/** + * @brief Get a list of compressors supported in the current build. + * + * @return The comma separated string with the list of compressor names + * supported. + * + * This function does not leak, so you should not free() the returned + * list. + * + * This function should always succeed. + */ +BLOSC_EXPORT const char* blosc2_list_compressors(void); + + +/** + * @brief Get the version of Blosc in string format. + * + * @return The string with the current Blosc version. + * Useful for dynamic libraries. + */ +BLOSC_EXPORT const char* blosc2_get_version_string(void); + + +/** + * @brief Get info from compression libraries included in the current build. + * + * @param compname The compressor name that you want info from. + * @param complib The pointer to a string where the + * compression library name, if available, will be put. + * @param version The pointer to a string where the + * compression library version, if available, will be put. + * + * @warning You are in charge of the @p complib and @p version strings, + * you should free() them so as to avoid leaks. + * + * @return The code for the compression library (>=0). If it is not supported, + * this function returns -1. + */ +BLOSC_EXPORT int blosc2_get_complib_info(const char* compname, char** complib, + char** version); + + +/** + * @brief Free possible memory temporaries and thread resources. Use this + * when you are not going to use Blosc for a long while. + * + * @return A 0 if succeeds, in case of problems releasing the resources, + * it returns a negative number. + */ +BLOSC_EXPORT int blosc2_free_resources(void); + + +/** + * @brief Get information about a compressed buffer, namely the number of + * uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also + * returns the @p blocksize (which is used internally for doing the + * compression by blocks). + * + * @remark Equivalent to function #blosc2_cbuffer_sizes. + * + * @param cbuffer The buffer of compressed data. + * @param nbytes The pointer where the number of uncompressed bytes will be put. + * @param cbytes The pointer where the number of compressed bytes will be put. + * @param blocksize The pointer where the block size will be put. + * + * You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a + * compressed buffer for this call to work. + * + * This function should always succeed. + */ +BLOSC_EXPORT void blosc1_cbuffer_sizes(const void* cbuffer, size_t* nbytes, + size_t* cbytes, size_t* blocksize); +/** + * @brief Get information about a compressed buffer, namely the number of + * uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also + * returns the @p blocksize (which is used internally for doing the + * compression by blocks). + * + * @param cbuffer The buffer of compressed data. + * @param nbytes The pointer where the number of uncompressed bytes will be put. + * @param cbytes The pointer where the number of compressed bytes will be put. + * @param blocksize The pointer where the block size will be put. + * + * @note: if any of the nbytes, cbytes or blocksize is NULL, it will not be returned. + * + * You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a + * compressed buffer for this call to work. + * + * @return On failure, returns negative value. + */ +BLOSC_EXPORT int blosc2_cbuffer_sizes(const void* cbuffer, int32_t* nbytes, + int32_t* cbytes, int32_t* blocksize); + +/** + * @brief Checks that the compressed buffer starting at @p cbuffer of length @p cbytes + * may contain valid blosc compressed data, and that it is safe to call + * blosc1_decompress/blosc1_getitem. + * On success, returns 0 and sets @p nbytes to the size of the uncompressed data. + * This does not guarantee that the decompression function won't return an error, + * but does guarantee that it is safe to attempt decompression. + * + * @param cbuffer The buffer of compressed data. + * @param cbytes The number of compressed bytes. + * @param nbytes The pointer where the number of uncompressed bytes will be put. + * + * @return On failure, returns negative value. + */ +BLOSC_EXPORT int blosc1_cbuffer_validate(const void* cbuffer, size_t cbytes, + size_t* nbytes); + +/** + * @brief Get information about a compressed buffer, namely the type size + * (@p typesize), as well as some internal @p flags. + * + * @param cbuffer The buffer of compressed data. + * @param typesize The pointer where the type size will be put. + * @param flags The pointer of the integer where the additional info is encoded. + * The @p flags is a set of bits, where the currently used ones are: + * * bit 0: whether the shuffle filter has been applied or not + * * bit 1: whether the internal buffer is a pure memcpy or not + * * bit 2: whether the bitshuffle filter has been applied or not + * * bit 3: whether the delta coding filter has been applied or not + * + * You can use the @p BLOSC_DOSHUFFLE, @p BLOSC_DOBITSHUFFLE, @p BLOSC_DODELTA + * and @p BLOSC_MEMCPYED symbols for extracting the interesting bits + * (e.g. @p flags & @p BLOSC_DOSHUFFLE says whether the buffer is byte-shuffled + * or not). + * + * This function should always succeed. + */ +BLOSC_EXPORT void blosc1_cbuffer_metainfo(const void* cbuffer, size_t* typesize, + int* flags); + + +/** + * @brief Get information about a compressed buffer, namely the internal + * Blosc format version (@p version) and the format for the internal + * Lempel-Ziv compressor used (@p versionlz). + * + * @param cbuffer The buffer of compressed data. + * @param version The pointer where the Blosc format version will be put. + * @param versionlz The pointer where the Lempel-Ziv version will be put. + * + * This function should always succeed. + */ +BLOSC_EXPORT void blosc2_cbuffer_versions(const void* cbuffer, int* version, + int* versionlz); + + +/** + * @brief Get the compressor library/format used in a compressed buffer. + * + * @param cbuffer The buffer of compressed data. + * + * @return The string identifying the compressor library/format used. + * + * This function should always succeed. + */ +BLOSC_EXPORT const char* blosc2_cbuffer_complib(const void* cbuffer); + +/********************************************************************* + Structures and functions related with user-defined input/output. +*********************************************************************/ + +enum { + BLOSC2_IO_FILESYSTEM = 0, + BLOSC2_IO_FILESYSTEM_MMAP = 1, + BLOSC_IO_LAST_BLOSC_DEFINED = 2, // sentinel + BLOSC_IO_LAST_REGISTERED = 32, // sentinel +}; + +enum { + BLOSC2_IO_BLOSC_DEFINED = 32, + BLOSC2_IO_REGISTERED = 160, + BLOSC2_IO_USER_DEFINED = 256 +}; + +typedef void* (*blosc2_open_cb)(const char *urlpath, const char *mode, void *params); +typedef int (*blosc2_close_cb)(void *stream); +typedef int64_t (*blosc2_size_cb)(void *stream); +typedef int64_t (*blosc2_write_cb)(const void *ptr, int64_t size, int64_t nitems, int64_t position, void *stream); +typedef int64_t (*blosc2_read_cb)(void **ptr, int64_t size, int64_t nitems, int64_t position, void *stream); +typedef int (*blosc2_truncate_cb)(void *stream, int64_t size); +typedef int (*blosc2_destroy_cb)(void *params); + + +/* + * Input/Output callbacks. + */ +typedef struct { + uint8_t id; + //!< The IO identifier. + char* name; + //!< The IO name. + bool is_allocation_necessary; + //!< If true, the caller needs to allocate data for the read function (ptr argument). If false, the read function + //!< takes care of memory allocation and stores the address in the allocated_ptr argument. + blosc2_open_cb open; + //!< The IO open callback. + blosc2_close_cb close; + //!< The IO close callback. + blosc2_size_cb size; + //!< The IO size callback. + blosc2_write_cb write; + //!< The IO write callback. + blosc2_read_cb read; + //!< The IO read callback. + blosc2_truncate_cb truncate; + //!< The IO truncate callback. + blosc2_destroy_cb destroy; + //!< The IO destroy callback (called in the end when finished with the schunk). +} blosc2_io_cb; + + +/* + * Input/Output parameters. + */ +typedef struct { + uint8_t id; + const char *name; + //!< The IO identifier. + void *params; + //!< The IO parameters. +} blosc2_io; + +static const blosc2_io BLOSC2_IO_DEFAULTS = { + /* .id = */ BLOSC2_IO_FILESYSTEM, + /* .name = */ "filesystem", + /* .params = */ NULL, +}; + + +/** + * @brief Register a user-defined input/output callbacks in Blosc. + * + * @param io The callbacks API to register. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_register_io_cb(const blosc2_io_cb *io); + +/** + * @brief Get a user-defined input/output callback in Blosc. + * + * @param id The id of the callback to get. + * + * @return A pointer containing the desired callback if success. Else a NULL pointer is returned. + */ +BLOSC_EXPORT blosc2_io_cb *blosc2_get_io_cb(uint8_t id); + +/********************************************************************* + Structures and functions related with contexts. +*********************************************************************/ + +typedef struct blosc2_context_s blosc2_context; /* opaque type */ + +typedef struct { + int (*init)(void * config, blosc2_context* cctx, blosc2_context* dctx); + //!< Initialize tuner. Keep in mind dctx may be NULL. This should memcpy the cctx->tuner_params. + int (*next_blocksize)(blosc2_context * context); + //!< Only compute the next blocksize. Only it is executed if tuner is not initialized. + int (*next_cparams)(blosc2_context * context); + //!< Compute the next cparams. Only is executed if tuner is initialized. + int (*update)(blosc2_context * context, double ctime); + //!< Update the tuner parameters. + int (*free)(blosc2_context * context); + //!< Free the tuner. + int id; + //!< The tuner id + char *name; + //!< The tuner name +} blosc2_tuner; + + +/** + * @brief Register locally a user-defined tuner in Blosc. + * + * @param tuner The tuner to register. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_register_tuner(blosc2_tuner *tuner); + + +/** + * @brief The parameters for a prefilter function. + * + */ +typedef struct { + void *user_data; // user-provided info (optional) + const uint8_t *input; // the input buffer + uint8_t *output; // the output buffer + int32_t output_size; // the output size (in bytes) + int32_t output_typesize; // the output typesize + int32_t output_offset; // offset to reach the start of the output buffer + int64_t nchunk; // the current nchunk in associated schunk (if exists; if not -1) + int32_t nblock; // the current nblock in associated chunk + int32_t tid; // thread id + uint8_t *ttmp; // a temporary that is able to hold several blocks for the output and is private for each thread + size_t ttmp_nbytes; // the size of the temporary in bytes + blosc2_context *ctx; // the compression context +} blosc2_prefilter_params; + +/** + * @brief The parameters for a postfilter function. + * + */ +typedef struct { + void *user_data; // user-provided info (optional) + const uint8_t *input; // the input buffer + uint8_t *output; // the output buffer + int32_t size; // the input size (in bytes) + int32_t typesize; // the input typesize + int32_t offset; // offset to reach the start of the input buffer + int64_t nchunk; // the current nchunk in associated schunk (if exists; if not -1) + int32_t nblock; // the current nblock in associated chunk + int32_t tid; // thread id + uint8_t *ttmp; // a temporary that is able to hold several blocks for the output and is private for each thread + size_t ttmp_nbytes; // the size of the temporary in bytes + blosc2_context *ctx; // the decompression context +} blosc2_postfilter_params; + +/** + * @brief The type of the prefilter function. + * + * If the function call is successful, the return value should be 0; else, a negative value. + */ +typedef int (*blosc2_prefilter_fn)(blosc2_prefilter_params* params); + +/** + * @brief The type of the postfilter function. + * + * If the function call is successful, the return value should be 0; else, a negative value. + */ +typedef int (*blosc2_postfilter_fn)(blosc2_postfilter_params* params); + +/** + * @brief The parameters for creating a context for compression purposes. + * + * In parenthesis it is shown the default value used internally when a 0 + * (zero) in the fields of the struct is passed to a function. + */ +typedef struct { + uint8_t compcode; + //!< The compressor codec. + uint8_t compcode_meta; + //!< The metadata for the compressor codec. + uint8_t clevel; + //!< The compression level (5). + int use_dict; + //!< Use dicts or not when compressing (only for ZSTD). + int32_t typesize; + //!< The type size (8). + int16_t nthreads; + //!< The number of threads to use internally (1). + int32_t blocksize; + //!< The requested size of the compressed blocks (0 means automatic). + int32_t splitmode; + //!< Whether the blocks should be split or not. + void* schunk; + //!< The associated schunk, if any (NULL). + uint8_t filters[BLOSC2_MAX_FILTERS]; + //!< The (sequence of) filters. + uint8_t filters_meta[BLOSC2_MAX_FILTERS]; + //!< The metadata for filters. + blosc2_prefilter_fn prefilter; + //!< The prefilter function. + blosc2_prefilter_params *preparams; + //!< The prefilter parameters. + void *tuner_params; + //!< Tune configuration. + int tuner_id; + //!< The tuner id. + bool instr_codec; + //!< Whether the codec is instrumented or not + void *codec_params; + //!< User defined parameters for the codec + void *filter_params[BLOSC2_MAX_FILTERS]; + //!< User defined parameters for the filters +} blosc2_cparams; + +/** + * @brief Default struct for compression params meant for user initialization. + */ +static const blosc2_cparams BLOSC2_CPARAMS_DEFAULTS = { + BLOSC_BLOSCLZ, 0, 5, 0, 8, 1, 0, + BLOSC_FORWARD_COMPAT_SPLIT, NULL, + {0, 0, 0, 0, 0, BLOSC_SHUFFLE}, + {0, 0, 0, 0, 0, 0}, + NULL, NULL, NULL, 0, 0, + NULL, {NULL, NULL, NULL, NULL, NULL, NULL} + }; + + +/** + @brief The parameters for creating a context for decompression purposes. + + In parenthesis it is shown the default value used internally when a 0 + (zero) in the fields of the struct is passed to a function. + */ +typedef struct { + int16_t nthreads; + //!< The number of threads to use internally (1). + void* schunk; + //!< The associated schunk, if any (NULL). + blosc2_postfilter_fn postfilter; + //!< The postfilter function. + blosc2_postfilter_params *postparams; + //!< The postfilter parameters. +} blosc2_dparams; + +/** + * @brief Default struct for decompression params meant for user initialization. + */ +static const blosc2_dparams BLOSC2_DPARAMS_DEFAULTS = {1, NULL, NULL, NULL}; + +/** + * @brief Create a context for @a *_ctx() compression functions. + * + * @param cparams The blosc2_cparams struct with the compression parameters. + * + * @return A pointer to the new context. NULL is returned if this fails. + * + * @note This supports the same environment variables than #blosc2_compress + * for overriding the programmatic compression values. + * + * @sa #blosc2_compress + */ +BLOSC_EXPORT blosc2_context* blosc2_create_cctx(blosc2_cparams cparams); + +/** + * @brief Create a context for *_ctx() decompression functions. + * + * @param dparams The blosc2_dparams struct with the decompression parameters. + * + * @return A pointer to the new context. NULL is returned if this fails. + * + * @note This supports the same environment variables than #blosc2_decompress + * for overriding the programmatic decompression values. + * + * @sa #blosc2_decompress + * + */ +BLOSC_EXPORT blosc2_context* blosc2_create_dctx(blosc2_dparams dparams); + +/** + * @brief Free the resources associated with a context. + * + * @param context The context to free. + * + * This function should always succeed and is valid for contexts meant for + * both compression and decompression. + */ +BLOSC_EXPORT void blosc2_free_ctx(blosc2_context* context); + +/** + * @brief Create a @p cparams associated to a context. + * + * @param ctx The context from where to extract the compression parameters. + * @param cparams The pointer where the compression params will be stored. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_ctx_get_cparams(blosc2_context *ctx, blosc2_cparams *cparams); + +/** + * @brief Create a @p dparams associated to a context. + * + * @param ctx The context from where to extract the decompression parameters. + * @param dparams The pointer where the decompression params will be stored. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_ctx_get_dparams(blosc2_context *ctx, blosc2_dparams *dparams); + +/** + * @brief Set a maskout so as to avoid decompressing specified blocks. + * + * @param ctx The decompression context to update. + * + * @param maskout The boolean mask for the blocks where decompression + * is to be avoided. + * + * @remark The maskout is valid for contexts *only* meant for decompressing + * a chunk via #blosc2_decompress_ctx. Once a call to #blosc2_decompress_ctx + * is done, this mask is reset so that next call to #blosc2_decompress_ctx + * will decompress the whole chunk. + * + * @param nblocks The number of blocks in maskout above. + * + * @return If success, a 0 is returned. An error is signaled with a negative int. + * + */ +BLOSC_EXPORT int blosc2_set_maskout(blosc2_context *ctx, bool *maskout, int nblocks); + +/** + * @brief Compress a block of data in the @p src buffer and returns the size of + * compressed block. + * + * @remark Compression is memory safe and guaranteed not to write @p dest + * more than what is specified in @p destsize. + * There is not a minimum for @p src buffer size @p nbytes. + * + * @warning The @p src buffer and the @p dest buffer can not overlap. + * + * @param clevel The desired compression level and must be a number + * between 0 (no compression) and 9 (maximum compression). + * @param doshuffle Specifies whether the shuffle compression preconditioner + * should be applied or not. #BLOSC_NOFILTER means not applying filters, + * #BLOSC_SHUFFLE means applying shuffle at a byte level and + * #BLOSC_BITSHUFFLE at a bit level (slower but *may* achieve better + * compression). + * @param typesize Is the number of bytes for the atomic type in binary + * @p src buffer. This is mainly useful for the shuffle preconditioner. + * For implementation reasons, only a 1 < typesize < 256 will allow the + * shuffle filter to work. When typesize is not in this range, shuffle + * will be silently disabled. + * @param src The buffer containing the data to compress. + * @param srcsize The number of bytes to compress in the @p src buffer. + * @param dest The buffer where the compressed data will be put, + * must have at least the size of @p destsize. + * @param destsize The size of the dest buffer. Blosc + * guarantees that if you set @p destsize to, at least, + * (@p nbytes + #BLOSC2_MAX_OVERHEAD), the compression will always succeed. + * + * @return The number of bytes compressed. + * If @p src buffer cannot be compressed into @p destsize, the return + * value is zero and you should discard the contents of the @p dest + * buffer. A negative return value means that either a parameter is not correct + * or that an internal error happened. Set the BLOSC_TRACE environment variable + * for getting more info on what is happening. If the error is not related with + * wrong params, please report it back together with the buffer data causing this, + * as well as the compression params used. +*/ +/* + * Environment variables + * _____________________ + * + * *blosc2_compress()* honors different environment variables to control + * internal parameters without the need of doing that programmatically. + * Here are the ones supported: + * + * **BLOSC_CLEVEL=(INTEGER)**: This will overwrite the @p clevel parameter + * before the compression process starts. + * + * **BLOSC_SHUFFLE=[NOSHUFFLE | SHUFFLE | BITSHUFFLE]**: This will + * overwrite the *doshuffle* parameter before the compression process + * starts. + * + * **BLOSC_DELTA=(1|0)**: This will call *blosc2_set_delta()^* before the + * compression process starts. + * + * **BLOSC_TYPESIZE=(INTEGER)**: This will overwrite the *typesize* + * parameter before the compression process starts. + * + * **BLOSC_COMPRESSOR=[BLOSCLZ | LZ4 | LZ4HC | ZLIB | ZSTD]**: + * This will call #blosc_set_compressor before the compression process starts. + * + * **BLOSC_NTHREADS=(INTEGER)**: This will call + * #blosc_set_nthreads before the compression process + * starts. + * + * **BLOSC_SPLITMODE=(ALWAYS | NEVER | AUTO | FORWARD_COMPAT)**: + * This will call #blosc1_set_splitmode() before the compression process starts. + * + * **BLOSC_BLOCKSIZE=(INTEGER)**: This will call + * #blosc_set_blocksize before the compression process starts. + * *NOTE:* The blocksize is a critical parameter with + * important restrictions in the allowed values, so use this with care. + * + * **BLOSC_NOLOCK=(ANY VALUE)**: This will call #blosc2_compress_ctx under + * the hood, with the *compressor*, *blocksize* and + * *numinternalthreads* parameters set to the same as the last calls to + * #blosc1_set_compressor, #blosc1_set_blocksize and + * #blosc2_set_nthreads. *BLOSC_CLEVEL*, *BLOSC_SHUFFLE*, *BLOSC_DELTA* and + * *BLOSC_TYPESIZE* environment vars will also be honored. + * + */ +BLOSC_EXPORT int blosc2_compress(int clevel, int doshuffle, int32_t typesize, + const void* src, int32_t srcsize, void* dest, + int32_t destsize); + + +/** + * @brief Decompress a block of compressed data in @p src, put the result in + * @p dest and returns the size of the decompressed block. + * + * @warning The @p src buffer and the @p dest buffer can not overlap. + * + * @remark Decompression is memory safe and guaranteed not to write the @p dest + * buffer more than what is specified in @p destsize. + * + * @remark In case you want to keep under control the number of bytes read from + * source, you can call #blosc1_cbuffer_sizes first to check whether the + * @p nbytes (i.e. the number of bytes to be read from @p src buffer by this + * function) in the compressed buffer is ok with you. + * + * @param src The buffer to be decompressed. + * @param srcsize The size of the buffer to be decompressed. + * @param dest The buffer where the decompressed data will be put. + * @param destsize The size of the @p dest buffer. + * + * @return The number of bytes decompressed. + * If an error occurs, e.g. the compressed data is corrupted or the + * output buffer is not large enough, then a negative value + * will be returned instead. +*/ +/* + * Environment variables + * _____________________ + * + * *blosc1_decompress* honors different environment variables to control + * internal parameters without the need of doing that programmatically. + * Here are the ones supported: + * + * **BLOSC_NTHREADS=(INTEGER)**: This will call + * *blosc_set_nthreads(BLOSC_NTHREADS)* before the proper decompression + * process starts. + * + * **BLOSC_NOLOCK=(ANY VALUE)**: This will call *blosc2_decompress_ctx* + * under the hood, with the *numinternalthreads* parameter set to the + * same value as the last call to *blosc2_set_nthreads*. + * + */ +BLOSC_EXPORT int blosc2_decompress(const void* src, int32_t srcsize, + void* dest, int32_t destsize); + +/** + * @brief Context interface to Blosc compression. This does not require a call + * to #blosc2_init and can be called from multithreaded applications + * without the global lock being used, so allowing Blosc be executed + * simultaneously in those scenarios. + * + * @param context A blosc2_context struct with the different compression params. + * @param src The buffer containing the data to be compressed. + * @param srcsize The number of bytes to be compressed from the @p src buffer. + * @param dest The buffer where the compressed data will be put. + * @param destsize The size in bytes of the @p dest buffer. + * + * @return The number of bytes compressed. + * If @p src buffer cannot be compressed into @p destsize, the return + * value is zero and you should discard the contents of the @p dest + * buffer. A negative return value means that an internal error happened. + * It could happen that context is not meant for compression (which is stated in stderr). + * Otherwise, please report it back together with the buffer data causing this + * and compression settings. + */ +BLOSC_EXPORT int blosc2_compress_ctx( + blosc2_context* context, const void* src, int32_t srcsize, void* dest, + int32_t destsize); + + +/** + * @brief Context interface to Blosc decompression. This does not require a + * call to #blosc2_init and can be called from multithreaded + * applications without the global lock being used, so allowing Blosc + * be executed simultaneously in those scenarios. + * + * @param context The blosc2_context struct with the different compression params. + * @param src The buffer of compressed data. + * @param srcsize The length of buffer of compressed data. + * @param dest The buffer where the decompressed data will be put. + * @param destsize The size in bytes of the @p dest buffer. + * + * @warning The @p src buffer and the @p dest buffer can not overlap. + * + * @remark Decompression is memory safe and guaranteed not to write the @p dest + * buffer more than what is specified in @p destsize. + * + * @remark In case you want to keep under control the number of bytes read from + * source, you can call #blosc1_cbuffer_sizes first to check the @p nbytes + * (i.e. the number of bytes to be read from @p src buffer by this function) + * in the compressed buffer. + * + * @remark If #blosc2_set_maskout is called prior to this function, its + * @p block_maskout parameter will be honored for just *one single* shot; + * i.e. the maskout in context will be automatically reset to NULL, so + * mask won't be used next time (unless #blosc2_set_maskout is called again). + * + * @return The number of bytes decompressed (i.e. the maskout blocks are not + * counted). If an error occurs, e.g. the compressed data is corrupted, + * @p destsize is not large enough or context is not meant for decompression, + * then a negative value will be returned instead. + */ +BLOSC_EXPORT int blosc2_decompress_ctx(blosc2_context* context, const void* src, + int32_t srcsize, void* dest, int32_t destsize); + +/** + * @brief Create a chunk made of zeros. + * + * @param cparams The compression parameters. + * @param nbytes The size (in bytes) of the chunk. + * @param dest The buffer where the data chunk will be put. + * @param destsize The size (in bytes) of the @p dest buffer; + * must be BLOSC_EXTENDED_HEADER_LENGTH at least. + * + * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). + * If negative, there has been an error and @p dest is unusable. + * */ +BLOSC_EXPORT int blosc2_chunk_zeros(blosc2_cparams cparams, int32_t nbytes, + void* dest, int32_t destsize); + + +/** + * @brief Create a chunk made of nans. + * + * @param cparams The compression parameters; + * only 4 bytes (float) and 8 bytes (double) are supported. + * @param nbytes The size (in bytes) of the chunk. + * @param dest The buffer where the data chunk will be put. + * @param destsize The size (in bytes) of the @p dest buffer; + * must be BLOSC_EXTENDED_HEADER_LENGTH at least. + * + * @note Whether the NaNs are floats or doubles will be given by the typesize. + * + * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). + * If negative, there has been an error and @p dest is unusable. + * */ +BLOSC_EXPORT int blosc2_chunk_nans(blosc2_cparams cparams, int32_t nbytes, + void* dest, int32_t destsize); + + +/** + * @brief Create a chunk made of repeated values. + * + * @param cparams The compression parameters. + * @param nbytes The size (in bytes) of the chunk. + * @param dest The buffer where the data chunk will be put. + * @param destsize The size (in bytes) of the @p dest buffer. + * @param repeatval A pointer to the repeated value (little endian). + * The size of the value is given by @p cparams.typesize param. + * + * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH + typesize). + * If negative, there has been an error and @p dest is unusable. + * */ +BLOSC_EXPORT int blosc2_chunk_repeatval(blosc2_cparams cparams, int32_t nbytes, + void* dest, int32_t destsize, const void* repeatval); + + +/** + * @brief Create a chunk made of uninitialized values. + * + * @param cparams The compression parameters. + * @param nbytes The size (in bytes) of the chunk. + * @param dest The buffer where the data chunk will be put. + * @param destsize The size (in bytes) of the @p dest buffer; + * must be BLOSC_EXTENDED_HEADER_LENGTH at least. + * + * @return The number of bytes compressed (BLOSC_EXTENDED_HEADER_LENGTH). + * If negative, there has been an error and @p dest is unusable. + * */ +BLOSC_EXPORT int blosc2_chunk_uninit(blosc2_cparams cparams, int32_t nbytes, + void* dest, int32_t destsize); + + +/** + * @brief Context interface counterpart for #blosc1_getitem. + * + * @param context Context pointer. + * @param src The compressed buffer from data will be decompressed. + * @param srcsize Compressed buffer length. + * @param start The position of the first item (of @p typesize size) from where data + * will be retrieved. + * @param nitems The number of items (of @p typesize size) that will be retrieved. + * @param dest The buffer where the decompressed data retrieved will be put. + * @param destsize Output buffer length. + * + * @return The number of bytes copied to @p dest or a negative value if + * some error happens. + */ +BLOSC_EXPORT int blosc2_getitem_ctx(blosc2_context* context, const void* src, + int32_t srcsize, int start, int nitems, void* dest, + int32_t destsize); + + +/********************************************************************* + Super-chunk related structures and functions. +*********************************************************************/ + +#define BLOSC2_MAX_METALAYERS 16 +#define BLOSC2_METALAYER_NAME_MAXLEN 31 + +// Allow for a reasonable number of vl metalayers +// max is 64 * 1024 due to msgpack map 16 in frame +// mem usage 8 * 1024 entries for blosc2_schunk.vlmetalayers[] is 64 KB +#define BLOSC2_MAX_VLMETALAYERS (8 * 1024) +#define BLOSC2_VLMETALAYERS_NAME_MAXLEN BLOSC2_METALAYER_NAME_MAXLEN + +/** + * @brief This struct is meant for holding storage parameters for a + * for a blosc2 container, allowing to specify, for example, how to interpret + * the contents included in the schunk. + */ +typedef struct { + bool contiguous; + //!< Whether the chunks are contiguous or sparse. + char* urlpath; + //!< The path for persistent storage. If NULL, that means in-memory. + blosc2_cparams* cparams; + //!< The compression params when creating a schunk. + //!< If NULL, sensible defaults are used depending on the context. + blosc2_dparams* dparams; + //!< The decompression params when creating a schunk. + //!< If NULL, sensible defaults are used depending on the context. + blosc2_io *io; + //!< Input/output backend. +} blosc2_storage; + +/** + * @brief Default struct for #blosc2_storage meant for user initialization. + */ +static const blosc2_storage BLOSC2_STORAGE_DEFAULTS = {false, NULL, NULL, NULL, NULL}; + +/** + * @brief Get default struct for compression params meant for user initialization. + */ +BLOSC_EXPORT blosc2_cparams blosc2_get_blosc2_cparams_defaults(void); + +/** + * @brief Get default struct for decompression params meant for user initialization. + */ +BLOSC_EXPORT blosc2_dparams blosc2_get_blosc2_dparams_defaults(void); + +/** + * @brief Get default struct for #blosc2_storage meant for user initialization. + */ +BLOSC_EXPORT blosc2_storage blosc2_get_blosc2_storage_defaults(void); + +/** + * @brief Get default struct for #blosc2_io meant for user initialization. + */ +BLOSC_EXPORT blosc2_io blosc2_get_blosc2_io_defaults(void); + +/** + * @brief Get default struct for #blosc2_stdio_mmap meant for user initialization. + */ +BLOSC_EXPORT blosc2_stdio_mmap blosc2_get_blosc2_stdio_mmap_defaults(void); + +typedef struct blosc2_frame_s blosc2_frame; /* opaque type */ + +/** + * @brief This struct is meant to store metadata information inside + * a #blosc2_schunk, allowing to specify, for example, how to interpret + * the contents included in the schunk. + */ +typedef struct blosc2_metalayer { + char* name; //!< The metalayer identifier for Blosc client (e.g. Blosc2 NDim). + uint8_t* content; //!< The serialized (msgpack preferably) content of the metalayer. + int32_t content_len; //!< The length in bytes of the content. +} blosc2_metalayer; + +/** + * @brief This struct is the standard container for Blosc 2 compressed data. + * + * This is essentially a container for Blosc 1 chunks of compressed data, + * and it allows to overcome the 32-bit limitation in Blosc 1. Optionally, + * a #blosc2_frame can be attached so as to store the compressed chunks contiguously. + */ +typedef struct blosc2_schunk { + uint8_t version; + uint8_t compcode; + //!< The default compressor. Each chunk can override this. + uint8_t compcode_meta; + //!< The default compressor metadata. Each chunk can override this. + uint8_t clevel; + //!< The compression level and other compress params. + uint8_t splitmode; + //!< The split mode. + int32_t typesize; + //!< The type size. + int32_t blocksize; + //!< The requested size of the compressed blocks (0; meaning automatic). + int32_t chunksize; + //!< Size of each chunk. 0 if not a fixed chunksize. + uint8_t filters[BLOSC2_MAX_FILTERS]; + //!< The (sequence of) filters. 8-bit per filter. + uint8_t filters_meta[BLOSC2_MAX_FILTERS]; + //!< Metadata for filters. 8-bit per meta-slot. + int64_t nchunks; + //!< Number of chunks in super-chunk. + int64_t current_nchunk; + //!< The current chunk that is being accessed + int64_t nbytes; + //!< The data size (uncompressed). + int64_t cbytes; + //!< The data size + chunks header size (compressed). + uint8_t** data; + //!< Pointer to chunk data pointers buffer. + size_t data_len; + //!< Length of the chunk data pointers buffer. + blosc2_storage* storage; + //!< Pointer to storage info. + blosc2_frame* frame; + //!< Pointer to frame used as store for chunks. + //! BLOSC2_METALAYER_NAME_MAXLEN) { + BLOSC_TRACE_ERROR("Metalayers cannot be larger than %d chars.", BLOSC2_METALAYER_NAME_MAXLEN); + return BLOSC2_ERROR_INVALID_PARAM; + } + + if (schunk == NULL) { + BLOSC_TRACE_ERROR("Schunk must not be NUll."); + return BLOSC2_ERROR_INVALID_PARAM; + } + + for (int nmetalayer = 0; nmetalayer < schunk->nmetalayers; nmetalayer++) { + if (strcmp(name, schunk->metalayers[nmetalayer]->name) == 0) { + return nmetalayer; + } + } + return BLOSC2_ERROR_NOT_FOUND; +} + +/** + * @brief Add content into a new metalayer. + * + * @param schunk The super-chunk to which the metalayer should be added. + * @param name The name of the metalayer. + * @param content The content of the metalayer. + * @param content_len The length of the content. + * + * @return If successful, the index of the new metalayer. Else, return a negative value. + */ +BLOSC_EXPORT int blosc2_meta_add(blosc2_schunk *schunk, const char *name, uint8_t *content, + int32_t content_len); + +/** + * @brief Update the content of an existing metalayer. + * + * @param schunk The frame containing the metalayer. + * @param name The name of the metalayer to be updated. + * @param content The new content of the metalayer. + * @param content_len The length of the content. + * + * @note Contrarily to #blosc2_meta_add the updates to metalayers + * are automatically serialized into a possible attached frame. + * + * @return If successful, the index of the metalayer. Else, return a negative value. + */ +BLOSC_EXPORT int blosc2_meta_update(blosc2_schunk *schunk, const char *name, uint8_t *content, + int32_t content_len); + +/** + * @brief Get the content out of a metalayer. + * + * @param schunk The frame containing the metalayer. + * @param name The name of the metalayer. + * @param content The pointer where the content will be put. + * @param content_len The length of the content. + * + * @warning The @p **content receives a malloc'ed copy of the content. + * The user is responsible of freeing it. + * + * @note This function is inlined and available even when not linking with libblosc2. + * + * @return If successful, the index of the new metalayer. Else, return a negative value. + */ +static inline int blosc2_meta_get(blosc2_schunk *schunk, const char *name, uint8_t **content, + int32_t *content_len) { + int nmetalayer = blosc2_meta_exists(schunk, name); + if (nmetalayer < 0) { + BLOSC_TRACE_WARNING("Metalayer \"%s\" not found.", name); + return nmetalayer; + } + *content_len = schunk->metalayers[nmetalayer]->content_len; + *content = (uint8_t*)malloc((size_t)*content_len); + memcpy(*content, schunk->metalayers[nmetalayer]->content, (size_t)*content_len); + return nmetalayer; +} + + +/********************************************************************* + Variable-length metalayers functions. +*********************************************************************/ + +/** + * @brief Find whether the schunk has a variable-length metalayer or not. + * + * @param schunk The super-chunk from which the variable-length metalayer will be checked. + * @param name The name of the variable-length metalayer to be checked. + * + * @return If successful, return the index of the variable-length metalayer. Else, return a negative value. + */ +BLOSC_EXPORT int blosc2_vlmeta_exists(blosc2_schunk *schunk, const char *name); + +/** + * @brief Add content into a new variable-length metalayer. + * + * @param schunk The super-chunk to which the variable-length metalayer should be added. + * @param name The name of the variable-length metalayer. + * @param content The content to be added. + * @param content_len The length of the content. + * @param cparams The parameters for compressing the variable-length metalayer content. If NULL, + * the `BLOSC2_CPARAMS_DEFAULTS` will be used. + * + * @return If successful, the index of the new variable-length metalayer. Else, return a negative value. + */ +BLOSC_EXPORT int blosc2_vlmeta_add(blosc2_schunk *schunk, const char *name, + uint8_t *content, int32_t content_len, + blosc2_cparams *cparams); + +/** + * @brief Update the content of an existing variable-length metalayer. + * + * @param schunk The super-chunk containing the variable-length metalayer. + * @param name The name of the variable-length metalayer to be updated. + * @param content The new content of the variable-length metalayer. + * @param content_len The length of the content. + * @param cparams The parameters for compressing the variable-length metalayer content. If NULL, + * the `BLOSC2_CPARAMS_DEFAULTS` will be used. + * + * @return If successful, the index of the variable-length metalayer. Else, return a negative value. + */ +BLOSC_EXPORT int blosc2_vlmeta_update(blosc2_schunk *schunk, const char *name, + uint8_t *content, int32_t content_len, + blosc2_cparams *cparams); + +/** + * @brief Get the content out of a variable-length metalayer. + * + * @param schunk The super-chunk containing the variable-length metalayer. + * @param name The name of the variable-length metalayer. + * @param content The pointer where the content will be put. + * @param content_len The pointer where the length of the content will be put. + * + * @warning The @p **content receives a malloc'ed copy of the content. + * The user is responsible of freeing it. + * + * @return If successful, the index of the new variable-length metalayer. Else, return a negative value. + */ +BLOSC_EXPORT int blosc2_vlmeta_get(blosc2_schunk *schunk, const char *name, + uint8_t **content, int32_t *content_len); + +/** + * @brief Delete the variable-length metalayer from the super-chunk. + * + * @param schunk The super-chunk containing the variable-length metalayer. + * @param name The name of the variable-length metalayer. + * + * @return If successful, the number of the variable-length metalayers in the super-chunk. Else, return a negative value. + */ +BLOSC_EXPORT int blosc2_vlmeta_delete(blosc2_schunk *schunk, const char *name); + +/** + * @brief Get a list of all the variable-length metalayer names. + * + * @param schunk The super-chunk containing the variable-length metalayers. + * @param names The pointer to a char** to store the name pointers. This should + * be of size *schunk->nvlmetalayers * sizeof(char*). + * + * @return The number of the variable-length metalayers in the super-chunk. + * This cannot fail unless the user does not pass a @p names which is large enough to + * keep pointers to all names, in which case funny things (seg faults and such) will happen. + */ +BLOSC_EXPORT int blosc2_vlmeta_get_names(blosc2_schunk *schunk, char **names); + + +/********************************************************************* + Time measurement utilities. +*********************************************************************/ + +#if defined(_WIN32) +/* For QueryPerformanceCounter(), etc. */ + #include +#elif defined(__MACH__) +#include +#include +#include +#elif defined(__unix__) +#if defined(__linux__) + #include + #else + #include + #endif +#else + #error Unable to detect platform. +#endif + +/* The type of timestamp used on this system. */ +#if defined(_WIN32) +typedef LARGE_INTEGER blosc_timestamp_t; +#else +typedef struct timespec blosc_timestamp_t; +#endif + +/* + * @brief Set a timestamp. + * + * @param timestamp + * + */ +BLOSC_EXPORT void blosc_set_timestamp(blosc_timestamp_t* timestamp); + +/* + * @brief Get the nanoseconds between 2 timestamps. + * + * @param start_time + * @param end_time + * + * @return The nanoseconds between start_time and end_time. + */ +BLOSC_EXPORT double blosc_elapsed_nsecs(blosc_timestamp_t start_time, + blosc_timestamp_t end_time); + +/* + * @brief Get the seconds between 2 timestamps. + * + * @param start_time + * @param end_time + * + * @return The seconds between start_time and end_time. + */ +BLOSC_EXPORT double blosc_elapsed_secs(blosc_timestamp_t start_time, + blosc_timestamp_t end_time); + + +/********************************************************************* + Low-level functions follows. Use them only if you are an expert! +*********************************************************************/ + +/** + * @brief Get the internal blocksize to be used during compression. 0 means + * that an automatic blocksize is computed internally. + * + * @return The size in bytes of the internal block size. + */ +BLOSC_EXPORT int blosc1_get_blocksize(void); + +/** + * @brief Force the use of a specific blocksize. If 0, an automatic + * blocksize will be used (the default). + * + * @warning The blocksize is a critical parameter with important + * restrictions in the allowed values, so use this with care. + */ +BLOSC_EXPORT void blosc1_set_blocksize(size_t blocksize); + + +/** + * @brief Set the split mode. + + * @param splitmode It can take the next values: + * BLOSC_FORWARD_COMPAT_SPLIT + * BLOSC_AUTO_SPLIT + * BLOSC_NEVER_SPLIT + * BLOSC_ALWAYS_SPLIT + * + * BLOSC_FORWARD_COMPAT offers reasonably forward compatibility, + * BLOSC_AUTO_SPLIT is for nearly optimal results (based on heuristics), + * BLOSC_NEVER_SPLIT and BLOSC_ALWAYS_SPLIT are for the user experimenting + * when trying to get best compression ratios and/or speed. + * + * If not called, the default mode is BLOSC_FORWARD_COMPAT_SPLIT. + * + * This function should always succeed. + */ +BLOSC_EXPORT void blosc1_set_splitmode(int splitmode); + + +/** + * @brief Get the offsets of a frame in a super-chunk. + * + * @param schunk The super-chunk containing the frame. + * + * @return If successful, return a pointer to a buffer of the decompressed offsets. + * The number of offsets is equal to schunk->nchunks; the user is + * responsible to free this buffer. Else, return a NULL value. + */ +BLOSC_EXPORT int64_t* blosc2_frame_get_offsets(blosc2_schunk *schunk); + + +/********************************************************************* + Structures and functions related with compression codecs. +*********************************************************************/ + +typedef int (* blosc2_codec_encoder_cb) (const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_cparams *cparams, const void* chunk); +typedef int (* blosc2_codec_decoder_cb) (const uint8_t *input, int32_t input_len, uint8_t *output, int32_t output_len, + uint8_t meta, blosc2_dparams *dparams, const void* chunk); + +typedef struct { + uint8_t compcode; + //!< The codec identifier. + char *compname; + //!< The codec name. + uint8_t complib; + //!< The codec library format. + uint8_t version; + //!< The codec version. + blosc2_codec_encoder_cb encoder; + //!< The codec encoder that is used during compression. + blosc2_codec_decoder_cb decoder; + //!< The codec decoder that is used during decompression. +} blosc2_codec; + +/** + * @brief Register locally a user-defined codec in Blosc. + * + * @param codec The codec to register. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_register_codec(blosc2_codec *codec); + + +/********************************************************************* + Structures and functions related with filters plugins. +*********************************************************************/ + +typedef int (* blosc2_filter_forward_cb) (const uint8_t *, uint8_t *, int32_t, uint8_t, blosc2_cparams *, + uint8_t); +typedef int (* blosc2_filter_backward_cb) (const uint8_t *, uint8_t *, int32_t, uint8_t, blosc2_dparams *, + uint8_t); + +/** + * @brief The parameters for a user-defined filter. + */ +typedef struct { + uint8_t id; + //!< The filter identifier. + char * name; + //!< The filter name. + uint8_t version; + //!< The filter version. + blosc2_filter_forward_cb forward; + //!< The filter function that is used during compression. + blosc2_filter_backward_cb backward; + //!< The filter function that is used during decompression. +} blosc2_filter; + +/** + * @brief Register locally a user-defined filter in Blosc. + * + * @param filter The filter to register. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_register_filter(blosc2_filter *filter); + +/********************************************************************* + Directory utilities. +*********************************************************************/ + +/* + * @brief Remove a directory and its files. + * + * @param path The directory to remove. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_remove_dir(const char *path); + +/* + * @brief Remove a file or a directory given by path. + * + * @param path The file or directory to remove. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_remove_urlpath(const char *path); + +/* + * @brief Rename a file or a directory given by old_urlpath to new_path. + * + * @param old_urlpath The original path to the directory or file. + * @param new_path The new path to the directory or file. + * + * @return 0 if succeeds. Else a negative code is returned. + */ +BLOSC_EXPORT int blosc2_rename_urlpath(char* old_urlpath, char* new_path); + + +/********************************************************************* + Index utilities. +*********************************************************************/ + +/* + * @brief Convert a sequential index into a multidimensional index + */ +BLOSC_EXPORT void blosc2_unidim_to_multidim(uint8_t ndim, int64_t *shape, int64_t i, int64_t *index); + +/* + * @brief Convert a multidimensional index into a sequential index + */ +BLOSC_EXPORT void blosc2_multidim_to_unidim(const int64_t *index, int8_t ndim, const int64_t *strides, int64_t *i); + +/* + * @brief Get the unidimensional chunk indexes needed to get a slice of a schunk or a b2nd array + * + * @param schunk The super-chunk (of b2nd array or not). + * @param start Index (0-based if it is a schunk) where the slice begins. + * @param stop The first index (0-based if it is a schunk) that is not in the selected slice. + * @param chunks_idx The pointer to the buffer where the indexes will be written. It is the user responsibility + * to free the buffer. + * + * @return The number of chunks needed to get the slice. If some problem is + * detected, a negative code is returned instead. + */ +BLOSC_EXPORT int blosc2_get_slice_nchunks(blosc2_schunk* schunk, int64_t *start, int64_t *stop, int64_t **chunks_idx); + + +/********************************************************************* + Private functions, these are here for convenience, + and are not meant to be included in public docs +*********************************************************************/ + +// Private function needed in b2nd.h for deserializing meta +static inline void swap_store(void *dest, const void *pa, int size) { + uint8_t *pa_ = (uint8_t *) pa; + uint8_t *pa2_ = (uint8_t*)malloc((size_t) size); + int i = 1; /* for big/little endian detection */ + char *p = (char *) &i; + + if (p[0] == 1) { + /* little endian */ + switch (size) { + case 8: + pa2_[0] = pa_[7]; + pa2_[1] = pa_[6]; + pa2_[2] = pa_[5]; + pa2_[3] = pa_[4]; + pa2_[4] = pa_[3]; + pa2_[5] = pa_[2]; + pa2_[6] = pa_[1]; + pa2_[7] = pa_[0]; + break; + case 4: + pa2_[0] = pa_[3]; + pa2_[1] = pa_[2]; + pa2_[2] = pa_[1]; + pa2_[3] = pa_[0]; + break; + case 2: + pa2_[0] = pa_[1]; + pa2_[1] = pa_[0]; + break; + case 1: + pa2_[0] = pa_[0]; + break; + default: + fprintf(stderr, "Unhandled nitems: %d\n", size); + } + } + memcpy(dest, pa2_, size); + free(pa2_); +} + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_H */ diff --git a/bltest/include/blosc2/blosc2-common.h b/bltest/include/blosc2/blosc2-common.h new file mode 100644 index 00000000..dd93afff --- /dev/null +++ b/bltest/include/blosc2/blosc2-common.h @@ -0,0 +1,80 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (c) 2021 Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_BLOSC2_BLOSC2_COMMON_H +#define BLOSC_BLOSC2_BLOSC2_COMMON_H + +#include "blosc2-export.h" + +#include +#include + +// For shutting up stupid compiler warning about some 'unused' variables in GCC +#ifdef __GNUC__ +#define BLOSC_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#define BLOSC_UNUSED_VAR __attribute__ ((unused)) +#else +#define BLOSC_UNUSED_VAR +#endif // __GNUC__ + +// For shutting up compiler warning about unused parameters +#define BLOSC_UNUSED_PARAM(x) ((void)(x)) + +/* Use inlined functions for supported systems */ +#if defined(_MSC_VER) && !defined(__cplusplus) /* Visual Studio */ + #define inline __inline /* Visual C is not C99, but supports some kind of inline */ +#endif + + +/* Define the __SSE2__ symbol if compiling with Visual C++ and + targeting the minimum architecture level supporting SSE2. + Other compilers define this as expected and emit warnings + when it is re-defined. */ +#if !defined(__SSE2__) && defined(_MSC_VER) && \ + (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2)) + #define __SSE2__ +#endif + +/* + * Detect if the architecture is fine with unaligned access. + */ +#if !defined(BLOSC_STRICT_ALIGN) +#define BLOSC_STRICT_ALIGN +#if defined(__i386__) || defined(__386) || defined (__amd64) /* GNU C, Sun Studio */ +#undef BLOSC_STRICT_ALIGN +#elif defined(__i486__) || defined(__i586__) || defined(__i686__) /* GNU C */ +#undef BLOSC_STRICT_ALIGN +#elif defined(_M_IX86) || defined(_M_X64) /* Intel, MSVC */ +#undef BLOSC_STRICT_ALIGN +#elif defined(__386) +#undef BLOSC_STRICT_ALIGN +#elif defined(_X86_) /* MinGW */ +#undef BLOSC_STRICT_ALIGN +#elif defined(__I86__) /* Digital Mars */ +#undef BLOSC_STRICT_ALIGN +/* Modern ARM systems (like ARM64) should support unaligned access + quite efficiently. */ +#elif defined(__ARM_FEATURE_UNALIGNED) && defined(__ARM64_ARCH_8__) +#undef BLOSC_STRICT_ALIGN +#elif defined(_ARCH_PPC) || defined(__PPC__) +/* Modern PowerPC systems (like POWER8) should support unaligned access + quite efficiently. */ +#undef BLOSC_STRICT_ALIGN +#endif +#endif + +#if defined(__SSE2__) + #include +#endif +#if defined(__AVX2__) || defined(__AVX512F__) || defined (__AVX512BW__) + #include +#endif + +#endif /* BLOSC_BLOSC2_BLOSC2_COMMON_H */ diff --git a/bltest/include/blosc2/blosc2-export.h b/bltest/include/blosc2/blosc2-export.h new file mode 100644 index 00000000..e5daed22 --- /dev/null +++ b/bltest/include/blosc2/blosc2-export.h @@ -0,0 +1,48 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (c) 2021 Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_BLOSC2_BLOSC2_EXPORT_H +#define BLOSC_BLOSC2_BLOSC2_EXPORT_H + +/* Macros for specifying exported symbols. + BLOSC_EXPORT is used to decorate symbols that should be + exported by the blosc shared library. + BLOSC_NO_EXPORT is used to decorate symbols that should NOT + be exported by the blosc shared library. +*/ +#if defined(BLOSC_SHARED_LIBRARY) + #if defined(_MSC_VER) + #define BLOSC_EXPORT __declspec(dllexport) + #elif (defined(__GNUC__) && __GNUC__ >= 4) || defined(__clang__) + #if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) + #define BLOSC_EXPORT __attribute__((dllexport)) + #else + #define BLOSC_EXPORT __attribute__((visibility("default"))) + #endif /* defined(_WIN32) || defined(__CYGWIN__) */ + #else + #error Cannot determine how to define BLOSC_EXPORT for this compiler. + #endif +#else + #define BLOSC_EXPORT +#endif /* defined(BLOSC_SHARED_LIBRARY) */ + +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__MINGW32__) + #define BLOSC_NO_EXPORT __attribute__((visibility("hidden"))) +#else + #define BLOSC_NO_EXPORT +#endif /* (defined(__GNUC__) || defined(__clang__)) && !defined(__MINGW32__) */ + +/* When testing, export everything to make it easier to implement tests. */ +#if defined(BLOSC_TESTING) + #undef BLOSC_NO_EXPORT + #define BLOSC_NO_EXPORT BLOSC_EXPORT +#endif /* defined(BLOSC_TESTING) */ + +#endif /* BLOSC_BLOSC2_BLOSC2_EXPORT_H */ diff --git a/bltest/include/blosc2/blosc2-stdio.h b/bltest/include/blosc2/blosc2-stdio.h new file mode 100644 index 00000000..3d7c3aeb --- /dev/null +++ b/bltest/include/blosc2/blosc2-stdio.h @@ -0,0 +1,117 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (c) 2021 Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_BLOSC2_BLOSC2_STDIO_H +#define BLOSC_BLOSC2_BLOSC2_STDIO_H + +#include "blosc2-export.h" + +#if defined(_MSC_VER) +#include +#else +#include +#endif + +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + FILE *file; +} blosc2_stdio_file; + +BLOSC_EXPORT void *blosc2_stdio_open(const char *urlpath, const char *mode, void* params); +BLOSC_EXPORT int blosc2_stdio_close(void *stream); +BLOSC_EXPORT int64_t blosc2_stdio_size(void *stream); +BLOSC_EXPORT int64_t blosc2_stdio_write(const void *ptr, int64_t size, int64_t nitems, int64_t position, void *stream); +BLOSC_EXPORT int64_t blosc2_stdio_read(void **ptr, int64_t size, int64_t nitems, int64_t position, void *stream); +BLOSC_EXPORT int blosc2_stdio_truncate(void *stream, int64_t size); +BLOSC_EXPORT int blosc2_stdio_destroy(void* params); + + +/** + * @brief Parameters for memory-mapped I/O. You can use the blosc2_schunk_open*_udio functions to memory-map existing + * schunk files from disk. To create a new schunk which is backed up by a memory-mapped file on disk, set the io member + * of the #blosc2_storage struct (see test_mmap for examples). Please note that memory-mapped I/O is only available for + * cframes and not sframes. + */ +typedef struct { + /* Arguments of the mapping */ + const char* mode; + //!< The opening mode of the memory-mapped file (r, r+, w+ or c) similar to Numpy's np.memmap + //!< (https://numpy.org/doc/stable/reference/generated/numpy.memmap.html). Set to r if the file should only be read, + //!< r+ if you want to extend data to an existing file, w+ to create a new file and c to use an existing file as basis + //!< but keep all modifications in-memory. On Windows, the size of the mapping cannot change in the c mode. + int64_t initial_mapping_size; + //!< The initial size of the memory mapping used as a large enough write buffer for the r+, w+ and c modes (for + //!< Windows, only the r+ and w+ modes). On Windows, this will also be the size of the file while the file is opened. + //!< It will be truncated to the target size when the file is closed (e.g., when the schunk is destroyed). + bool needs_free; + //!< Indicates whether this object should be freed in the blosc2_destroy_cb callback (set to true if the + //!< blosc2_stdio_mmap struct was created on the heap). + + /* Internal attributes of the mapping */ + char* addr; + //!< The starting address of the mapping. + char* urlpath; + //!< The path to the file which is associated with this object. + int64_t file_size; + //!< The size of the file. + int64_t mapping_size; + //!< The size of the mapping (mapping_size >= file_size). + bool is_memory_only; + //!< Whether the mapping is only in-memory and changes are not reflected to the file on disk (c mode). + FILE* file; + //!< The underlying file handle. + int fd; + //!< The underlying file descriptor. + int64_t access_flags; + //!< The access attributes for the memory pages. + int64_t map_flags; + //!< The attributes of the mapping. +#if defined(_WIN32) + HANDLE mmap_handle; + //!< The Windows handle to the memory mapping. +#endif +} blosc2_stdio_mmap; + +/** + * @brief Default struct for memory-mapped I/O for user initialization. + */ +static const blosc2_stdio_mmap BLOSC2_STDIO_MMAP_DEFAULTS = { + "r", (1 << 30), false, NULL, NULL, -1, -1, false, NULL, -1, -1, -1 +#if defined(_WIN32) + , INVALID_HANDLE_VALUE +#endif +}; + +BLOSC_EXPORT void *blosc2_stdio_mmap_open(const char *urlpath, const char *mode, void* params); +BLOSC_EXPORT int blosc2_stdio_mmap_close(void *stream); +BLOSC_EXPORT int64_t blosc2_stdio_mmap_size(void *stream); +BLOSC_EXPORT int64_t blosc2_stdio_mmap_write( + const void *ptr, int64_t size, int64_t nitems, int64_t position, void *stream); +BLOSC_EXPORT int64_t blosc2_stdio_mmap_read(void **ptr, int64_t size, int64_t nitems, int64_t position, void *stream); +BLOSC_EXPORT int blosc2_stdio_mmap_truncate(void *stream, int64_t size); +BLOSC_EXPORT int blosc2_stdio_mmap_destroy(void* params); + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_BLOSC2_STDIO_H */ diff --git a/bltest/include/blosc2/codecs-registry.h b/bltest/include/blosc2/codecs-registry.h new file mode 100644 index 00000000..e33df531 --- /dev/null +++ b/bltest/include/blosc2/codecs-registry.h @@ -0,0 +1,50 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (c) 2021 Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_BLOSC2_CODECS_REGISTRY_H +#define BLOSC_BLOSC2_CODECS_REGISTRY_H + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + BLOSC_CODEC_NDLZ = 32, + //!< Simple Lempel-Ziv compressor for NDim data. Experimental, mainly for teaching purposes. + BLOSC_CODEC_ZFP_FIXED_ACCURACY = 33, + //!< ZFP compressor for fixed accuracy mode. The desired accuracy is set in `compcode_meta`. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/codecs/zfp/README.md + BLOSC_CODEC_ZFP_FIXED_PRECISION = 34, + //!< ZFP compressor for fixed precision. The desired precision is set in `compcode_meta`. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/codecs/zfp/README.md + BLOSC_CODEC_ZFP_FIXED_RATE = 35, + //!< ZFP compressor for fixed precision. The desired rate is set in `compcode_meta`. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/codecs/zfp/README.md + BLOSC_CODEC_OPENHTJ2K = 36, + //!< OpenHTJ2K compressor for JPEG 2000 HT. + //!< See https://github.com/Blosc/blosc2_openhtj2k + BLOSC_CODEC_GROK = 37, + //!< Grok compressor for JPEG 2000. + //!< See https://github.com/Blosc/blosc2_grok +}; + +void register_codecs(void); + +// For dynamically loaded codecs +typedef struct { + char *encoder; + char *decoder; +} codec_info; + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_CODECS_REGISTRY_H */ diff --git a/bltest/include/blosc2/filters-registry.h b/bltest/include/blosc2/filters-registry.h new file mode 100644 index 00000000..22061dcf --- /dev/null +++ b/bltest/include/blosc2/filters-registry.h @@ -0,0 +1,50 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (c) 2021 Blosc Development Team + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_BLOSC2_FILTERS_REGISTRY_H +#define BLOSC_BLOSC2_FILTERS_REGISTRY_H + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + BLOSC_FILTER_NDCELL = 32, + //!< Simple filter for grouping NDim cell data together. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/filters/ndcell/README.md + BLOSC_FILTER_NDMEAN = 33, + //!< Simple filter for replacing content of a NDim cell with its mean value. + //!< See https://github.com/Blosc/c-blosc2/blob/main/plugins/filters/ndmean/README.md + BLOSC_FILTER_BYTEDELTA_BUGGY = 34, + // buggy version. See #524 + BLOSC_FILTER_BYTEDELTA = 35, + //!< Byteshuffle + delta. The typesize should be specified in the `filters_meta` slot. + //!< Sometimes this can represent an advantage over + //!< @ref BLOSC_SHUFFLE or @ref BLOSC_BITSHUFFLE. + //!< See https://www.blosc.org/posts/bytedelta-enhance-compression-toolset/ + BLOSC_FILTER_INT_TRUNC = 36, + //!< Truncate int precision; positive values in `filters_meta` slot will keep bits; + //!< negative values will remove (set to zero) bits. + //!< This is similar to @ref BLOSC_TRUNC_PREC, but for integers instead of floating point data. +}; + +void register_filters(void); + +// For dynamically loaded filters +typedef struct { + char *forward; + char *backward; +} filter_info; + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_FILTERS_REGISTRY_H */ diff --git a/bltest/include/blosc2/tuners-registry.h b/bltest/include/blosc2/tuners-registry.h new file mode 100644 index 00000000..94300a54 --- /dev/null +++ b/bltest/include/blosc2/tuners-registry.h @@ -0,0 +1,37 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) + + See LICENSE.txt for details about copyright and rights to use. +**********************************************************************/ + +#ifndef BLOSC_BLOSC2_TUNERS_REGISTRY_H +#define BLOSC_BLOSC2_TUNERS_REGISTRY_H + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + BLOSC_BTUNE = 32, +}; + +void register_tuners(void); + +// For dynamically loaded tuners +typedef struct { + char *init; + char *next_blocksize; + char *next_cparams; + char *update; + char *free; +} tuner_info; + +#ifdef __cplusplus +} +#endif + +#endif /* BLOSC_BLOSC2_TUNERS_REGISTRY_H */ diff --git a/bltest/pyvenv.cfg b/bltest/pyvenv.cfg new file mode 100644 index 00000000..30c68c3a --- /dev/null +++ b/bltest/pyvenv.cfg @@ -0,0 +1,5 @@ +home = /usr/local/opt/python@3.12/bin +include-system-site-packages = false +version = 3.12.7 +executable = /usr/local/Cellar/python@3.12/3.12.7_1/Frameworks/Python.framework/Versions/3.12/bin/python3.12 +command = /usr/local/opt/python@3.12/bin/python3.12 -m venv /Users/jennefermaldonado/NSLS2/blop/bltest diff --git a/bltest/share/man/man1/isympy.1 b/bltest/share/man/man1/isympy.1 new file mode 100644 index 00000000..0ff96615 --- /dev/null +++ b/bltest/share/man/man1/isympy.1 @@ -0,0 +1,188 @@ +'\" -*- coding: us-ascii -*- +.if \n(.g .ds T< \\FC +.if \n(.g .ds T> \\F[\n[.fam]] +.de URL +\\$2 \(la\\$1\(ra\\$3 +.. +.if \n(.g .mso www.tmac +.TH isympy 1 2007-10-8 "" "" +.SH NAME +isympy \- interactive shell for SymPy +.SH SYNOPSIS +'nh +.fi +.ad l +\fBisympy\fR \kx +.if (\nx>(\n(.l/2)) .nr x (\n(.l/5) +'in \n(.iu+\nxu +[\fB-c\fR | \fB--console\fR] [\fB-p\fR ENCODING | \fB--pretty\fR ENCODING] [\fB-t\fR TYPE | \fB--types\fR TYPE] [\fB-o\fR ORDER | \fB--order\fR ORDER] [\fB-q\fR | \fB--quiet\fR] [\fB-d\fR | \fB--doctest\fR] [\fB-C\fR | \fB--no-cache\fR] [\fB-a\fR | \fB--auto\fR] [\fB-D\fR | \fB--debug\fR] [ +-- | PYTHONOPTIONS] +'in \n(.iu-\nxu +.ad b +'hy +'nh +.fi +.ad l +\fBisympy\fR \kx +.if (\nx>(\n(.l/2)) .nr x (\n(.l/5) +'in \n(.iu+\nxu +[ +{\fB-h\fR | \fB--help\fR} +| +{\fB-v\fR | \fB--version\fR} +] +'in \n(.iu-\nxu +.ad b +'hy +.SH DESCRIPTION +isympy is a Python shell for SymPy. It is just a normal python shell +(ipython shell if you have the ipython package installed) that executes +the following commands so that you don't have to: +.PP +.nf +\*(T< +>>> from __future__ import division +>>> from sympy import * +>>> x, y, z = symbols("x,y,z") +>>> k, m, n = symbols("k,m,n", integer=True) + \*(T> +.fi +.PP +So starting isympy is equivalent to starting python (or ipython) and +executing the above commands by hand. It is intended for easy and quick +experimentation with SymPy. For more complicated programs, it is recommended +to write a script and import things explicitly (using the "from sympy +import sin, log, Symbol, ..." idiom). +.SH OPTIONS +.TP +\*(T<\fB\-c \fR\*(T>\fISHELL\fR, \*(T<\fB\-\-console=\fR\*(T>\fISHELL\fR +Use the specified shell (python or ipython) as +console backend instead of the default one (ipython +if present or python otherwise). + +Example: isympy -c python + +\fISHELL\fR could be either +\&'ipython' or 'python' +.TP +\*(T<\fB\-p \fR\*(T>\fIENCODING\fR, \*(T<\fB\-\-pretty=\fR\*(T>\fIENCODING\fR +Setup pretty printing in SymPy. By default, the most pretty, unicode +printing is enabled (if the terminal supports it). You can use less +pretty ASCII printing instead or no pretty printing at all. + +Example: isympy -p no + +\fIENCODING\fR must be one of 'unicode', +\&'ascii' or 'no'. +.TP +\*(T<\fB\-t \fR\*(T>\fITYPE\fR, \*(T<\fB\-\-types=\fR\*(T>\fITYPE\fR +Setup the ground types for the polys. By default, gmpy ground types +are used if gmpy2 or gmpy is installed, otherwise it falls back to python +ground types, which are a little bit slower. You can manually +choose python ground types even if gmpy is installed (e.g., for testing purposes). + +Note that sympy ground types are not supported, and should be used +only for experimental purposes. + +Note that the gmpy1 ground type is primarily intended for testing; it the +use of gmpy even if gmpy2 is available. + +This is the same as setting the environment variable +SYMPY_GROUND_TYPES to the given ground type (e.g., +SYMPY_GROUND_TYPES='gmpy') + +The ground types can be determined interactively from the variable +sympy.polys.domains.GROUND_TYPES inside the isympy shell itself. + +Example: isympy -t python + +\fITYPE\fR must be one of 'gmpy', +\&'gmpy1' or 'python'. +.TP +\*(T<\fB\-o \fR\*(T>\fIORDER\fR, \*(T<\fB\-\-order=\fR\*(T>\fIORDER\fR +Setup the ordering of terms for printing. The default is lex, which +orders terms lexicographically (e.g., x**2 + x + 1). You can choose +other orderings, such as rev-lex, which will use reverse +lexicographic ordering (e.g., 1 + x + x**2). + +Note that for very large expressions, ORDER='none' may speed up +printing considerably, with the tradeoff that the order of the terms +in the printed expression will have no canonical order + +Example: isympy -o rev-lax + +\fIORDER\fR must be one of 'lex', 'rev-lex', 'grlex', +\&'rev-grlex', 'grevlex', 'rev-grevlex', 'old', or 'none'. +.TP +\*(T<\fB\-q\fR\*(T>, \*(T<\fB\-\-quiet\fR\*(T> +Print only Python's and SymPy's versions to stdout at startup, and nothing else. +.TP +\*(T<\fB\-d\fR\*(T>, \*(T<\fB\-\-doctest\fR\*(T> +Use the same format that should be used for doctests. This is +equivalent to '\fIisympy -c python -p no\fR'. +.TP +\*(T<\fB\-C\fR\*(T>, \*(T<\fB\-\-no\-cache\fR\*(T> +Disable the caching mechanism. Disabling the cache may slow certain +operations down considerably. This is useful for testing the cache, +or for benchmarking, as the cache can result in deceptive benchmark timings. + +This is the same as setting the environment variable SYMPY_USE_CACHE +to 'no'. +.TP +\*(T<\fB\-a\fR\*(T>, \*(T<\fB\-\-auto\fR\*(T> +Automatically create missing symbols. Normally, typing a name of a +Symbol that has not been instantiated first would raise NameError, +but with this option enabled, any undefined name will be +automatically created as a Symbol. This only works in IPython 0.11. + +Note that this is intended only for interactive, calculator style +usage. In a script that uses SymPy, Symbols should be instantiated +at the top, so that it's clear what they are. + +This will not override any names that are already defined, which +includes the single character letters represented by the mnemonic +QCOSINE (see the "Gotchas and Pitfalls" document in the +documentation). You can delete existing names by executing "del +name" in the shell itself. You can see if a name is defined by typing +"'name' in globals()". + +The Symbols that are created using this have default assumptions. +If you want to place assumptions on symbols, you should create them +using symbols() or var(). + +Finally, this only works in the top level namespace. So, for +example, if you define a function in isympy with an undefined +Symbol, it will not work. +.TP +\*(T<\fB\-D\fR\*(T>, \*(T<\fB\-\-debug\fR\*(T> +Enable debugging output. This is the same as setting the +environment variable SYMPY_DEBUG to 'True'. The debug status is set +in the variable SYMPY_DEBUG within isympy. +.TP +-- \fIPYTHONOPTIONS\fR +These options will be passed on to \fIipython (1)\fR shell. +Only supported when ipython is being used (standard python shell not supported). + +Two dashes (--) are required to separate \fIPYTHONOPTIONS\fR +from the other isympy options. + +For example, to run iSymPy without startup banner and colors: + +isympy -q -c ipython -- --colors=NoColor +.TP +\*(T<\fB\-h\fR\*(T>, \*(T<\fB\-\-help\fR\*(T> +Print help output and exit. +.TP +\*(T<\fB\-v\fR\*(T>, \*(T<\fB\-\-version\fR\*(T> +Print isympy version information and exit. +.SH FILES +.TP +\*(T<\fI${HOME}/.sympy\-history\fR\*(T> +Saves the history of commands when using the python +shell as backend. +.SH BUGS +The upstreams BTS can be found at \(lahttps://github.com/sympy/sympy/issues\(ra +Please report all bugs that you find in there, this will help improve +the overall quality of SymPy. +.SH "SEE ALSO" +\fBipython\fR(1), \fBpython\fR(1) diff --git a/bltest/share/man/man1/ttx.1 b/bltest/share/man/man1/ttx.1 new file mode 100644 index 00000000..bba23b5e --- /dev/null +++ b/bltest/share/man/man1/ttx.1 @@ -0,0 +1,225 @@ +.Dd May 18, 2004 +.\" ttx is not specific to any OS, but contrary to what groff_mdoc(7) +.\" seems to imply, entirely omitting the .Os macro causes 'BSD' to +.\" be used, so I give a zero-width space as its argument. +.Os \& +.\" The "FontTools Manual" argument apparently has no effect in +.\" groff 1.18.1. I think it is a bug in the -mdoc groff package. +.Dt TTX 1 "FontTools Manual" +.Sh NAME +.Nm ttx +.Nd tool for manipulating TrueType and OpenType fonts +.Sh SYNOPSIS +.Nm +.Bk +.Op Ar option ... +.Ek +.Bk +.Ar file ... +.Ek +.Sh DESCRIPTION +.Nm +is a tool for manipulating TrueType and OpenType fonts. It can convert +TrueType and OpenType fonts to and from an +.Tn XML Ns -based format called +.Tn TTX . +.Tn TTX +files have a +.Ql .ttx +extension. +.Pp +For each +.Ar file +argument it is given, +.Nm +detects whether it is a +.Ql .ttf , +.Ql .otf +or +.Ql .ttx +file and acts accordingly: if it is a +.Ql .ttf +or +.Ql .otf +file, it generates a +.Ql .ttx +file; if it is a +.Ql .ttx +file, it generates a +.Ql .ttf +or +.Ql .otf +file. +.Pp +By default, every output file is created in the same directory as the +corresponding input file and with the same name except for the +extension, which is substituted appropriately. +.Nm +never overwrites existing files; if necessary, it appends a suffix to +the output file name before the extension, as in +.Pa Arial#1.ttf . +.Ss "General options" +.Bl -tag -width ".Fl t Ar table" +.It Fl h +Display usage information. +.It Fl d Ar dir +Write the output files to directory +.Ar dir +instead of writing every output file to the same directory as the +corresponding input file. +.It Fl o Ar file +Write the output to +.Ar file +instead of writing it to the same directory as the +corresponding input file. +.It Fl v +Be verbose. Write more messages to the standard output describing what +is being done. +.It Fl a +Allow virtual glyphs ID's on compile or decompile. +.El +.Ss "Dump options" +The following options control the process of dumping font files +(TrueType or OpenType) to +.Tn TTX +files. +.Bl -tag -width ".Fl t Ar table" +.It Fl l +List table information. Instead of dumping the font to a +.Tn TTX +file, display minimal information about each table. +.It Fl t Ar table +Dump table +.Ar table . +This option may be given multiple times to dump several tables at +once. When not specified, all tables are dumped. +.It Fl x Ar table +Exclude table +.Ar table +from the list of tables to dump. This option may be given multiple +times to exclude several tables from the dump. The +.Fl t +and +.Fl x +options are mutually exclusive. +.It Fl s +Split tables. Dump each table to a separate +.Tn TTX +file and write (under the name that would have been used for the output +file if the +.Fl s +option had not been given) one small +.Tn TTX +file containing references to the individual table dump files. This +file can be used as input to +.Nm +as long as the referenced files can be found in the same directory. +.It Fl i +.\" XXX: I suppose OpenType programs (exist and) are also affected. +Don't disassemble TrueType instructions. When this option is specified, +all TrueType programs (glyph programs, the font program and the +pre-program) are written to the +.Tn TTX +file as hexadecimal data instead of +assembly. This saves some time and results in smaller +.Tn TTX +files. +.It Fl y Ar n +When decompiling a TrueType Collection (TTC) file, +decompile font number +.Ar n , +starting from 0. +.El +.Ss "Compilation options" +The following options control the process of compiling +.Tn TTX +files into font files (TrueType or OpenType): +.Bl -tag -width ".Fl t Ar table" +.It Fl m Ar fontfile +Merge the input +.Tn TTX +file +.Ar file +with +.Ar fontfile . +No more than one +.Ar file +argument can be specified when this option is used. +.It Fl b +Don't recalculate glyph bounding boxes. Use the values in the +.Tn TTX +file as is. +.El +.Sh "THE TTX FILE FORMAT" +You can find some information about the +.Tn TTX +file format in +.Pa documentation.html . +In particular, you will find in that file the list of tables understood by +.Nm +and the relations between TrueType GlyphIDs and the glyph names used in +.Tn TTX +files. +.Sh EXAMPLES +In the following examples, all files are read from and written to the +current directory. Additionally, the name given for the output file +assumes in every case that it did not exist before +.Nm +was invoked. +.Pp +Dump the TrueType font contained in +.Pa FreeSans.ttf +to +.Pa FreeSans.ttx : +.Pp +.Dl ttx FreeSans.ttf +.Pp +Compile +.Pa MyFont.ttx +into a TrueType or OpenType font file: +.Pp +.Dl ttx MyFont.ttx +.Pp +List the tables in +.Pa FreeSans.ttf +along with some information: +.Pp +.Dl ttx -l FreeSans.ttf +.Pp +Dump the +.Sq cmap +table from +.Pa FreeSans.ttf +to +.Pa FreeSans.ttx : +.Pp +.Dl ttx -t cmap FreeSans.ttf +.Sh NOTES +On MS\-Windows and MacOS, +.Nm +is available as a graphical application to which files can be dropped. +.Sh SEE ALSO +.Pa documentation.html +.Pp +.Xr fontforge 1 , +.Xr ftinfo 1 , +.Xr gfontview 1 , +.Xr xmbdfed 1 , +.Xr Font::TTF 3pm +.Sh AUTHORS +.Nm +was written by +.An -nosplit +.An "Just van Rossum" Aq just@letterror.com . +.Pp +This manual page was written by +.An "Florent Rougon" Aq f.rougon@free.fr +for the Debian GNU/Linux system based on the existing FontTools +documentation. It may be freely used, modified and distributed without +restrictions. +.\" For Emacs: +.\" Local Variables: +.\" fill-column: 72 +.\" sentence-end: "[.?!][]\"')}]*\\($\\| $\\| \\| \\)[ \n]*" +.\" sentence-end-double-space: t +.\" End: \ No newline at end of file diff --git a/src/blop/__init__.py b/src/blop/__init__.py index d2fc9547..8ba04d7c 100644 --- a/src/blop/__init__.py +++ b/src/blop/__init__.py @@ -1,5 +1,5 @@ from . import utils # noqa F401 -from ._version import __version__, __version_tuple__ # noqa: F401 +#from ._version import __version__, __version_tuple__ # noqa: F401 from .agent import Agent # noqa F401 from .dofs import DOF # noqa F401 from .objectives import Objective # noqa F401 diff --git a/src/blop/tests/test_sims.py b/src/blop/tests/test_sims.py index 5906ac11..bcdf98b7 100644 --- a/src/blop/tests/test_sims.py +++ b/src/blop/tests/test_sims.py @@ -3,6 +3,7 @@ from blop import DOF, Agent, Objective from blop.digestion import beam_stats_digestion from blop.sim import Beamline +import torch def test_kb_simulation(RE, db): @@ -37,3 +38,35 @@ def test_kb_simulation(RE, db): RE(agent.learn("qr", n=16)) RE(agent.learn("qei", n=4, iterations=4)) + + + +def test_nan_simulation(RE, db): + beamline = Beamline(name="bl") + beamline.det.noise.put(False) + + pos_x10 = 10000 + pos_y10 = 10000 + pos_x20 = 10000 + pos_y20 = 10000 + search_range = 0.6 + + dofs = [ + DOF(description="transfocator upstream x", device=beamline.kbv_dsv, search_domain=(pos_x10-search_range/2, pos_x10+search_range/2)), + DOF(description="transfocator downstream x", device=beamline.kbv_usv, search_domain=(pos_x20-search_range/2, pos_x20+search_range/2)), + DOF(description="transfocator upstream y", device=beamline.kbh_dsh, search_domain=(pos_y10-search_range/2, pos_y10+search_range/2)), + DOF(description="transfocator downstream y", device=beamline.kbh_ush, search_domain=(pos_y20-search_range/2, pos_y20+search_range/2)), + ] + + objectives = [ + Objective(name="bl_det_sum", target="max", trust_domain=(100000, np.inf)), + ] + + agent = Agent( + dofs=dofs, + objectives=objectives, + detectors=[beamline.det], + db=db, + ) + + RE(agent.learn("qr", n=16))