diff --git a/demos/write_streaming_sdds.ipynb b/demos/write_streaming_sdds.ipynb new file mode 100644 index 0000000..c5d4697 --- /dev/null +++ b/demos/write_streaming_sdds.ipynb @@ -0,0 +1,490 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true, + "pycharm": { + "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.089338Z", + "start_time": "2024-10-07T10:16:59.913133Z" + } + }, + "outputs": [], + "source": [ + "import pysdds\n", + "import logging, io\n", + "import numpy as np\n", + "\n", + "# To show logging messages in the notebook\n", + "logging.basicConfig()\n", + "logger = logging.getLogger()\n", + "logger.setLevel(logging.DEBUG)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "outputs": [], + "source": [ + "# # Load simple 1-page file\n", + "# sdds = pysdds.read('../tests/files/sources/xLinac.matrix')\n", + "# \n", + "# # Make it a bit more compact\n", + "# sdds.arrays = []\n", + "# sdds.columns = sdds.columns[:4]" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.095340Z", + "start_time": "2024-10-07T10:17:01.090839Z" + } + } + }, + { + "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SDDSFile: 0 pages, 2 parameters, 0 arrays, 2 columns, mode binary, endianness little\n", + " Page sizes: []\n", + " Column mem usage: 0.0000 MB\n" + ] + } + ], + "source": [ + "sdds = pysdds.SDDSFile(add_data_nm=True)\n", + "sdds.set_mode('binary')\n", + "\n", + "par_dbl = sdds.add_parameter('par1', 'double')\n", + "par_str = sdds.add_parameter('par2_str', 'string')\n", + "\n", + "col_dbl = sdds.add_column('col1', 'double', units='m')\n", + "col_str = sdds.add_column('col2_str', 'string')\n", + "assert sdds.n_parameters == 2\n", + "assert sdds.n_columns == 2\n", + "print(sdds.describe())" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.106342Z", + "start_time": "2024-10-07T10:17:01.096840Z" + } + }, + "execution_count": 3 + }, + { + "cell_type": "code", + "execution_count": 4, + "outputs": [], + "source": [ + "# Create a memory buffer for output - pysdds will automatically detect whether file or buffer is given\n", + "buf = io.BytesIO()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.111842Z", + "start_time": "2024-10-07T10:17:01.108342Z" + } + } + }, + { + "cell_type": "code", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:pysdds.writers.writers:Parameters: 2 of 2\n", + "DEBUG:pysdds.writers.writers:Parameter types: [dtype('float64'), None]\n", + "DEBUG:pysdds.writers.writers:Parameter lengths: [8, None]\n", + "DEBUG:pysdds.writers.writers:Columns: 2\n", + "DEBUG:pysdds.writers.writers:C types: [dtype('float64'), None]\n", + "DEBUG:pysdds.writers.writers:C lengths: [8, None]\n" + ] + } + ], + "source": [ + "# begin streaming by creating a stream writer bound to the SDDS object with no data but correct structure (this is \n", + "# already the case in our example)\n", + "sdds2 = sdds.copy(data=False)\n", + "sdds2.set_mode('binary')\n", + "stream_writer = sdds2.get_streaming_writer(filename=buf)" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.122845Z", + "start_time": "2024-10-07T10:17:01.114843Z" + } + }, + "execution_count": 5 + }, + { + "cell_type": "code", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:pysdds.writers.writers:Opening write stream to \"<_io.BytesIO object at 0x000001A211C3A610>\"\n", + "DEBUG:pysdds.writers.writers:Mode (binary), compression (None), endianness (little)\n", + "DEBUG:pysdds.writers.writers:Header write done in 0.074 ms\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SDDS5\n", + "!# little-endian\n", + "!# fixed-rowcount\n", + "!Generated by pysdds 0.3.0+17.gb7792ee.dirty\n", + "!Submit issues at github.com/nikitakuklev/pysdds\n", + "¶meter name=par1, type=double, &end\n", + "¶meter name=par2_str, type=string, &end\n", + "&column name=col1, type=double, units=m, &end\n", + "&column name=col2_str, type=string, &end\n", + "&data mode=binary, endian=little, no_row_counts=0, column_major_order=0, &end\n" + ] + } + ], + "source": [ + "# Write the header\n", + "stream_writer.begin()\n", + "print(buf.getvalue().decode('ascii'))" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.130846Z", + "start_time": "2024-10-07T10:17:01.124344Z" + } + }, + "execution_count": 6 + }, + { + "cell_type": "code", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:pysdds.writers.writers:Starting page 0 with 1000000 rows in FIXED COUNT MODE\n" + ] + } + ], + "source": [ + "# start new page by providing parameter and/or array data as lists (must match the order in the file)\n", + "stream_writer.new_page(parameter_data=[0.05, 'bla'])" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.137347Z", + "start_time": "2024-10-07T10:17:01.132346Z" + } + }, + "execution_count": 7 + }, + { + "cell_type": "code", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b'SDDS5\\n!# little-endian\\n!# fixed-rowcount\\n!Generated by pysdds 0.3.0+17.gb7792ee.dirty\\n!Submit issues at github.com/nikitakuklev/pysdds\\n¶meter name=par1, type=double, &end\\n¶meter name=par2_str, type=string, &end\\n&column name=col1, type=double, units=m, &end\\n&column name=col2_str, type=string, &end\\n&data mode=binary, endian=little, no_row_counts=0, column_major_order=0, &end\\n@B\\x0f\\x00\\x9a\\x99\\x99\\x99\\x99\\x99\\xa9?\\x03\\x00\\x00\\x00bla'\n" + ] + } + ], + "source": [ + "print(buf.getvalue())" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.144348Z", + "start_time": "2024-10-07T10:17:01.138847Z" + } + }, + "execution_count": 8 + }, + { + "cell_type": "code", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "DEBUG:pysdds.writers.writers:Single row [1.0, 'test'] converted to [array([1.]), array(['test'], dtype='\"\n", + "DEBUG:pysdds.readers.readers:Path opened as byte stream in 0.016 ms, size (None)\n", + "DEBUG:pysdds.readers.readers:Binary file endianness set to (little) from meta-command\n", + "DEBUG:pysdds.readers.readers:Fixed rowcount meta-command found - setting flag\n", + "DEBUG:pysdds.readers.readers:Binary file endianness set to (little) from data namelist\n", + "DEBUG:pysdds.readers.readers:Header parsed: 2 parameters, 0 arrays, 2 columns\n", + "DEBUG:pysdds.readers.readers:Parameters to parse: 2 of 2\n", + "DEBUG:pysdds.readers.readers:Parameter types: [dtype('float64'), None]\n", + "DEBUG:pysdds.readers.readers:Parameter lengths: [8, None]\n", + "DEBUG:pysdds.readers.readers:Columns to parse: 2\n", + "DEBUG:pysdds.readers.readers:Column types: [dtype('float64'), ]\n", + "DEBUG:pysdds.readers.readers:Column lengths: [8, None]\n", + "DEBUG:pysdds.readers.readers:All numeric: False\n", + "DEBUG:pysdds.readers.readers:Not all columns numeric, data is row order -> using slow sequential parser\n", + "DEBUG:pysdds.readers.readers:Page 0 size is 1000000 | byte_array=b'@B\\x0f\\x00'\n", + "INFO:pysdds.readers.readers:Encountered fixed rowcount file end at row 3 of 1000000\n", + "DEBUG:pysdds.readers.readers:Read in 10.676 ms, 1 pages, 3 rows, 2 parameters, 0/0 arrays, 2/2 columns\n", + "\n", + "DEBUG:pysdds.readers.readers:SDDSFile: 1 pages, 2 parameters, 0 arrays, 2 columns, mode binary, endianness little\n", + " Page sizes: [3]\n", + " Column mem usage: 0.0000 MB\n" + ] + } + ], + "source": [ + "sdds3 = pysdds.read(io.BufferedReader(io.BytesIO(result)))" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.210360Z", + "start_time": "2024-10-07T10:17:01.192857Z" + } + }, + "execution_count": 15 + }, + { + "cell_type": "code", + "outputs": [ + { + "data": { + "text/plain": " par1 par2_str\nPage \n0 0.05 bla", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
par1par2_str
Page
00.05bla
\n
" + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sdds3.pdf()" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:12.719372Z", + "start_time": "2024-10-07T10:17:12.711371Z" + } + }, + "execution_count": 17 + }, + { + "cell_type": "code", + "execution_count": 16, + "outputs": [ + { + "data": { + "text/plain": " col1 col2_str\n0 1.0 test\n1 1.1 foo\n2 1.5 bar", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
col1col2_str
01.0test
11.1foo
21.5bar
\n
" + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sdds3.cdf()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2024-10-07T10:17:01.229863Z", + "start_time": "2024-10-07T10:17:01.212360Z" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}