Merge branch 'ipython:main' into ssh_windows_support

.github/workflows/windows-ssh-image.yaml

@@ -0,0 +1,49 @@
+# because it's too unbelievably slow to install ssh server on Windows
+name: Build image for Windows SSH CI
+
+on:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ipyparallel-windows-ssh
+  TAG: 3.12-2022
+
+jobs:
+  build-and-push-image:
+    runs-on: windows-2022
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      #       - name: Extract metadata (tags, labels) for Docker
+      #         id: meta
+      #         uses: docker/metadata-action@v5
+      #         with:
+      #           labels: |
+      #             org.opencontainers.image.title=${{ env.IMAGE_NAME }}
+      #
+      #           images: ${{ github.repository_owner }}/${{ env.IMAGE_NAME }}
+      #           tags: |
+      #             type=raw,value=${{ env.TAG }}
+
+      - name: Build image
+        # can't use build-push-action on Windows
+        # https://github.com/docker/build-push-action/issues/18
+        run: |
+          docker build -t ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.TAG }} -f ci/ssh/win_base_Dockerfile ci/ssh
+      - name: Push image
+        run: |
+          docker push ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.TAG }}

.pre-commit-config.yaml

@@ -3,25 +3,38 @@ ci:
 
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.0
+    rev: v0.7.1
     hooks:
       - id: ruff
         args:
           - "--fix"
+        exclude_types:
+          - jupyter
       - id: ruff-format
+        exclude_types:
+          - jupyter
+  # run ruff via nbqa, which has better and configurable handling of cell magics
+  # than ruff itself
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.9.0
+    hooks:
+      - id: nbqa-ruff-format
+      - id: nbqa-ruff-check
+        args:
+          - --fix
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v4.0.0-alpha.8
     hooks:
       - id: prettier
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: end-of-file-fixer
       - id: check-case-conflict
       - id: check-executables-have-shebangs
       - id: requirements-txt-fixer
   - repo: https://github.com/pre-commit/mirrors-eslint
-    rev: v9.6.0
+    rev: v9.13.0
     hooks:
       - id: eslint
         files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx

benchmarks/async_benchmark.ipynb

@@ -6,16 +6,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from IPython.core.interactiveshell import InteractiveShell\n",
-    "from IPython.display import display, Markdown, SVG, HTML\n",
-    "import pandas as pd\n",
-    "import altair as alt\n",
-    "import re\n",
     "import pickle\n",
-    "from utils import seconds_to_ms, ms_to_seconds\n",
-    "from benchmark_result import get_benchmark_results, BenchmarkType, SchedulerType, get_broadcast_source, get_async_source\n",
-    "from benchmarks.utils import echo\n",
-    "from benchmarks.throughput import make_benchmark, make_multiple_message_benchmark"
+    "\n",
+    "import altair as alt\n",
+    "import pandas as pd\n",
+    "from benchmark_result import get_async_source"
    ]
   },
   {
@@ -24,8 +19,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "#benchmark_results = get_benchmark_results()\n",
-    "from benchmark_result import BenchmarkResult, Result \n",
+    "# benchmark_results = get_benchmark_results()\n",
     "with open('saved_results.pkl', 'rb') as saved_results:\n",
     "    benchmark_results = pickle.load(saved_results)"
    ]
@@ -118,9 +112,9 @@
    "outputs": [],
    "source": [
     "source = get_async_source(benchmark_results)\n",
-    "dview = pd.DataFrame(source['DirectView']) \n",
+    "dview = pd.DataFrame(source['DirectView'])\n",
     "dview['Scheduler name'] = 'DirectView'\n",
-    "dview['Speedup'] = 1\n"
+    "dview['Speedup'] = 1"
    ]
   },
   {
@@ -288,9 +282,13 @@
    "source": [
     "datas = []\n",
     "for scheduler_name, scheduler_results in source.items():\n",
-    "    data = pd.DataFrame(scheduler_results) \n",
+    "    data = pd.DataFrame(scheduler_results)\n",
     "    data['Scheduler name'] = scheduler_name\n",
-    "    data['Speedup'] = 1 if scheduler_name == 'DirectView' else dview['Duration in ms'] / data['Duration in ms']\n",
+    "    data['Speedup'] = (\n",
+    "        1\n",
+    "        if scheduler_name == 'DirectView'\n",
+    "        else dview['Duration in ms'] / data['Duration in ms']\n",
+    "    )\n",
     "    datas.append(data)\n",
     "data = pd.concat(datas)\n",
     "data"
@@ -386,8 +384,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "data['Messages per engine per second'] = round(data['Number of messages'] / data['Duration in ms']  * 1000, 2)\n",
-    "data['Total messages per second'] = round(((data['Number of messages'] * data['Number of engines']) / data['Duration in ms']) * 1000, 2)"
+    "data['Messages per engine per second'] = round(\n",
+    "    data['Number of messages'] / data['Duration in ms'] * 1000, 2\n",
+    ")\n",
+    "data['Total messages per second'] = round(\n",
+    "    ((data['Number of messages'] * data['Number of engines']) / data['Duration in ms'])\n",
+    "    * 1000,\n",
+    "    2,\n",
+    ")"
    ]
   },
   {
@@ -598,7 +602,7 @@
    ],
    "source": [
     "dview = data[data['Number of messages'] == 100]\n",
-    "dview['Number of engines'].unique()\n"
+    "dview['Number of engines'].unique()"
    ]
   },
   {
@@ -670,17 +674,13 @@
    ],
    "source": [
     "alt.Chart(dview).mark_line(point=True).encode(\n",
-    "    alt.X(\n",
-    "        'Number of engines',\n",
-    "        scale=alt.Scale(type='log', base=2)\n",
-    "    ),\n",
-    "    alt.Y(\n",
-    "        'Messages per engine per second',\n",
-    "        scale=alt.Scale(type='log')\n",
-    "    ),\n",
+    "    alt.X('Number of engines', scale=alt.Scale(type='log', base=2)),\n",
+    "    alt.Y('Messages per engine per second', scale=alt.Scale(type='log')),\n",
     "    color='Scheduler name:N',\n",
     "    tooltip='Messages per engine per second',\n",
-    ").configure_axis(labelFontSize=20, titleFontSize=20).properties(title='Runtime of apply using DirectView', width=1080).interactive().display(renderer='svg')"
+    ").configure_axis(labelFontSize=20, titleFontSize=20).properties(\n",
+    "    title='Runtime of apply using DirectView', width=1080\n",
+    ").interactive().display(renderer='svg')"
    ]
   },
   {
@@ -752,17 +752,13 @@
    ],
    "source": [
     "alt.Chart(dview).mark_line(point=True).encode(\n",
-    "    alt.X(\n",
-    "        'Number of engines',\n",
-    "        scale=alt.Scale(type='log', base=2)\n",
-    "    ),\n",
-    "    alt.Y(\n",
-    "        'Total messages per second',\n",
-    "        scale=alt.Scale(type='log')\n",
-    "    ),\n",
+    "    alt.X('Number of engines', scale=alt.Scale(type='log', base=2)),\n",
+    "    alt.Y('Total messages per second', scale=alt.Scale(type='log')),\n",
     "    color='Scheduler name:N',\n",
     "    tooltip='Total messages per second',\n",
-    ").configure_axis(labelFontSize=20, titleFontSize=20).properties(title='Runtime of apply using DirectView', width=1080).interactive().display(renderer='svg')"
+    ").configure_axis(labelFontSize=20, titleFontSize=20).properties(\n",
+    "    title='Runtime of apply using DirectView', width=1080\n",
+    ").interactive().display(renderer='svg')"
    ]
   },
   {
@@ -958,16 +954,15 @@
     "for scheduler_name in data['Scheduler name'].unique():\n",
     "    scheduler_data = data[data['Scheduler name'] == scheduler_name]\n",
     "    alt.Chart(scheduler_data).mark_line(point=True).encode(\n",
-    "        alt.X(\n",
-    "            'Number of messages',\n",
-    "            scale=alt.Scale(type='log')\n",
-    "        ),\n",
+    "        alt.X('Number of messages', scale=alt.Scale(type='log')),\n",
     "        alt.Y(\n",
     "            'Messages per engine per second',\n",
     "        ),\n",
     "        color='Number of engines:N',\n",
-    "        tooltip='Duration in ms',    \n",
-    "    ).configure_axis(labelFontSize=20, titleFontSize=20).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')"
+    "        tooltip='Duration in ms',\n",
+    "    ).configure_axis(labelFontSize=20, titleFontSize=20).properties(\n",
+    "        title=scheduler_name, width=800\n",
+    "    ).interactive().display(renderer='svg')"
    ]
   },
   {
@@ -1042,20 +1037,18 @@
     }
    ],
    "source": [
-    "data['combined']=  data['Scheduler name'] + ' ' + data['Number of engines'].astype(str)\n",
+    "data['combined'] = data['Scheduler name'] + ' ' + data['Number of engines'].astype(str)\n",
     "alt.Chart(data[data['Scheduler name'] != 'DirectView']).mark_line(point=True).encode(\n",
-    "    alt.X(\n",
-    "        'Number of messages',\n",
-    "            scale=alt.Scale(type='log')\n",
-    "    ),\n",
+    "    alt.X('Number of messages', scale=alt.Scale(type='log')),\n",
     "    alt.Y(\n",
     "        'Speedup',\n",
     "    ),\n",
     "    color='Number of engines:N',\n",
     "    strokeDash=alt.StrokeDash(shorthand='Scheduler name', legend=None),\n",
     "    tooltip='combined',\n",
-    "\n",
-    ").properties(title='schedulers vs directView scaling engines', width=800).interactive().display(renderer='svg')\n"
+    ").properties(\n",
+    "    title='schedulers vs directView scaling engines', width=800\n",
+    ").interactive().display(renderer='svg')"
    ]
   },
   {
@@ -1557,14 +1550,16 @@
     }
    ],
    "source": [
-    "for engine in data['Number of engines'].unique():    \n",
+    "for engine in data['Number of engines'].unique():\n",
     "    alt.Chart(data[data['Number of engines'] == engine]).mark_bar().encode(\n",
     "        x='Scheduler name',\n",
     "        y='Duration in ms',\n",
     "        color='Scheduler name:N',\n",
-    "        column='Number of messages:N',    \n",
-    "        tooltip='Duration in ms'\n",
-    "    ).properties(title=f'Runtime on {engine} engines:').interactive().display(renderer='svg')"
+    "        column='Number of messages:N',\n",
+    "        tooltip='Duration in ms',\n",
+    "    ).properties(title=f'Runtime on {engine} engines:').interactive().display(\n",
+    "        renderer='svg'\n",
+    "    )"
    ]
   }
  ],