Additional logging of cropping process

.cruft.json

@@ -1,6 +1,6 @@
 {
   "template": "https://github.com/arup-group/cookiecutter-pypackage.git",
-  "commit": "b7724521f898ab28f541d492aff8e2be2ed76a01",
+  "commit": "bb2bab415eded9682b2c1d465a281450ec922f1c",
   "checkout": null,
   "context": {
     "cookiecutter": {
@@ -15,11 +15,13 @@
       "project_short_description": "The Population activity Modeller (PAM) is a python API for activity sequence modelling.",
       "upload_pypi_package": "y",
       "upload_conda_package": "y",
+      "upload_aws_image": "n",
       "conda_channel": "city-modelling-lab",
       "command_line_interface": "y",
       "create_docker_file": "y",
       "create_author_file": "n",
       "create_jupyter_notebook_directory": "y",
+      "check_docs_accessibility_in_CI": "n",
       "open_source_license": "MIT license",
       "_template": "https://github.com/arup-group/cookiecutter-pypackage.git"
     }

.github/workflows/docs.yml

@@ -3,9 +3,16 @@ name: Build docs
 on:
   push:
     branches:
-      - "**"
+      - main
+
+  pull_request:
+    branches:
+      - main
     paths-ignore:
       - tests/**
+      - ".github/**/*"
+      - "!.github/workflows/docs.yml"
+
 
 jobs:
   docs-test:

.github/workflows/pr-ci.yml

@@ -11,9 +11,20 @@ on:
       - CONTRIBUTING.md
       - docs/**
       - mkdocs.yml
+      - ".github/**/*"
+      - "!.github/workflows/pr-ci.yml"
 
 jobs:
+  lint:
+    if: github.event.repository.private
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: pre-commit/action@v3.0.1
+
   test:
+    needs: lint
+    if: always() && (needs.lint.result  == 'success' || needs.lint.result  == 'skipped')
     strategy:
       matrix:
         os: [windows-latest, ubuntu-latest, macos-latest]
@@ -29,14 +40,16 @@ jobs:
       upload_to_codecov: false
 
   test-coverage:
+    needs: lint
+    if: always() && (needs.lint.result  == 'success' || needs.lint.result  == 'skipped')
     uses: arup-group/actions-city-modelling-lab/.github/workflows/python-install-lint-test.yml@main
     with:
       os: ubuntu-latest
       py3version: "12"
       notebook_kernel: pam
       lint: false
       pytest_args: 'tests/'  # ignore example notebooks
-      upload_to_codecov: true
+      upload_to_codecov: ${{ github.event.repository.visibility == 'public' }}  # only attempt to upload if the project is public
 
   memory-profile:
     uses: arup-group/actions-city-modelling-lab/.github/workflows/python-memory-profile.yml@main

.pa11yci

@@ -0,0 +1,24 @@
+{
+  "defaults": {
+    "reporters": [
+        "cli",
+      [
+        "pa11y-ci-reporter-html",
+        { "destination": "./reports/pa11y", "includeZeroIssues": false }
+      ]
+    ],
+    "timeout": 100000,
+    "wait": 2000,
+    "ignore": ["color-contrast"],
+    "runners": [
+      "htmlcs", "axe"
+    ],
+    "hideElements": "[id^='__codelineno'], .md-search__form, #__toc, clipboard-copy"
+  },
+  "standard": "WCAG2AA",
+  "comments": [
+    "Ignoring color-contrast due to https://github.com/pa11y/pa11y/issues/697.",
+    "Hiding `clipboard-copy` until https://github.com/danielfrg/mkdocs-jupyter/pull/206 is merged.",
+    "Hiding `[id^='__codelineno'], .md-search__form, #__toc` due to known false positives in mkdocs-material: https://github.com/squidfunk/mkdocs-material/discussions/4102"
+  ]
+}

CHANGELOG.md

@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 * Facility link snapping (#276).
+* Logging of the agents and households removed during (spatial) cropping (#285).
 
 ### Changed
 

CONTRIBUTING.md

@@ -36,19 +36,19 @@ Look at the [development guide in our documentation](https://arup-group.github.i
 To contribute changes:
 
 1. Fork the project on GitHub.
-2. Create a feature branch to work on in your fork (`git checkout -b new-fix-or-feature`).
-3. Test your changes using `pytest`.
-4. Commit your changes to the feature branch (you should have `pre-commit` installed to ensure your code is correctly formatted when you commit changes).
-5. Push the branch to GitHub (`git push origin new-fix-or-feature`).
-6. On GitHub, create a new [pull request](https://github.com/arup-group/pam/pull/new/main) from the feature branch.
+1. Create a feature branch to work on in your fork (`git checkout -b new-fix-or-feature`).
+1. Test your changes using `pytest`.
+1. Commit your changes to the feature branch (you should have `pre-commit` installed to ensure your code is correctly formatted when you commit changes).
+1. Push the branch to GitHub (`git push origin new-fix-or-feature`).
+1. On GitHub, create a new [pull request](https://github.com/arup-group/pam/pull/new/main) from the feature branch.
 
 ### Pull requests
 
 Before submitting a pull request, check whether you have:
 
-* Added your changes to `CHANGELOG.md`.
-* Added or updated documentation for your changes.
-* Added tests if you implemented new functionality.
+- Added your changes to `CHANGELOG.md`.
+- Added or updated documentation for your changes.
+- Added tests if you implemented new functionality.
 
 When opening a pull request, please provide a clear summary of your changes!
 

docs/activity_plans.md

@@ -135,4 +135,5 @@ This is resulting some difficult to follow logic (e.g., [][pam.activity.Plan.fil
 ## Plan cropping
 The [`pam.operations.cropping`](reference/pam/operations/cropping.md) module allows to spatially subset populations, by simplifying plan components that take place outside the "core" area.
 Any activities or legs that do not affect that core area are removed from the agents' plans, and agents with fully-external plans are removed from the population.
-Examples of using the module can be found in the [][plan-cropping] notebook.
+Examples of using the module can be found in the [][plan-cropping] notebook.
+Plan cropping now features basic logging on the input and then output population, quantifying the changes in agent and households as a result of the cropping. 

docs/static/extras.css

@@ -7,4 +7,25 @@ div.doc-contents:not(.first) {
 /* Allow tables to be horizontally scrollable. */
 .md-typeset__scrollwrap {
   overflow-x: auto;
-}
+}
+/* Improve contrast of text in rendered Jupyter notebook codeblocks. */
+.jp-Notebook {
+  --jp-cell-prompt-not-active-opacity: 1;
+}
+.jp-InputArea-editor {
+  --jp-cell-editor-background: var(--md-code-bg-color);
+}
+.highlight-ipynb {
+  --jp-mirror-editor-number-color: var(--md-code-hl-number-color);
+  --jp-mirror-editor-string-color: var(--md-code-hl-string-color);
+  --jp-mirror-editor-operator-color: var(--md-code-hl-operator-color);
+  --jp-mirror-editor-meta-color: var(--md-code-hl-operator-color);
+  --jp-mirror-editor-comment-color: var(--md-code-hl-comment-color);
+  --jp-mirror-editor-keyword-color: var(--md-code-hl-keyword-color);
+  --jp-mirror-editor-builtin-color: var(--md-code-hl-keyword-color);
+  --jp-mirror-editor-variable-2-color: var(--md-code-hl-punctuation-color);
+  --jp-mirror-editor-punctuation-color: var(--md-code-hl-punctuation-color);
+  --jp-mirror-editor-property-color: var(--md-code-hl-punctuation-color);
+  --jp-mirror-editor-variable-color: var(--md-code-hl-variable-color);
+  --jp-mirror-editor-def-color: var(--md-code-hl-constant-color);
+}

docs/static/hooks.py

@@ -1,3 +1,5 @@
+"""Hooks to run when building documentation."""
+
 import tempfile
 from pathlib import Path
 
@@ -10,8 +12,17 @@
 
 # Bump priority to ensure files are moved before jupyter notebook conversion takes place
 @mkdocs.plugins.event_priority(50)
-def on_files(files: list, config: dict, **kwargs):
-    """Link (1) top-level files to mkdocs files and (2) generate the python API documentation."""
+def on_files(files: list, config: dict, **kwargs) -> list:
+    """Link (1) top-level files to mkdocs files and (2) generate the python API documentation.
+
+    Args:
+        files (list): mkdocs file list.
+        config (dict): mkdocs config dictionary.
+        **kwargs: Automatic MKDocs hook inputs.
+
+    Returns:
+        list: Updated mkdocs file list.
+    """
     for file in sorted(Path("./examples").glob("*.ipynb")):
         files.append(_new_file(file, config))
         _get_nav_list(config["nav"], "Examples").append(file.as_posix())
@@ -48,7 +59,7 @@ def _new_file(path: Path, config: dict, src_dir: str = ".") -> File:
 
 
 def _api_gen(files: list, config: dict) -> dict:
-    """Project Python API generator
+    """Project Python API generator.
 
     Args:
         files (list): mkdocs file list.
@@ -78,6 +89,7 @@ def _py_to_md(filepath: Path, api_nav: dict, config: dict) -> File:
         filepath (Path): Path to python file relative to the package source code directory.
         api_nav (dict): Nested dictionary to fill with mkdocs navigation entries.
         config (Config): mkdocs config dictionary.
+
     Returns:
         File: mkdocs object that links the temp file to the docs directory, ready to be added to the mkdocs file list.
     """
@@ -112,7 +124,6 @@ def _update_nav(api_nav: dict, config: dict) -> None:
         api_nav (dict): Python API navigation tree.
         config (dict): mkdocs config dictionary (in which `nav` can be found).
     """
-
     api_reference_nav = {
         "Python API": [*api_nav.pop("top_level"), *[{k: v} for k, v in api_nav.items()]]
     }
@@ -121,6 +132,7 @@ def _update_nav(api_nav: dict, config: dict) -> None:
 
 def _get_nav_list(nav: list[dict | str], ref: str) -> list:
     """Get navigation entry sub-page list.
+
     Navigation list entries can be dictionaries or strings.
     Sub-list entries can then also be dictionaries or strings. E.g.,
 
@@ -144,6 +156,6 @@ def on_post_build(**kwargs):
     """After mkdocs has finished building the docs, remove the temporary directory of markdown files.
 
     Args:
-        config (Config): mkdocs config dictionary (unused).
+        **kwargs: Automatic MKDocs hook inputs.
     """
     TEMPDIR.cleanup()

mkdocs.yml

@@ -36,13 +36,12 @@ markdown_extensions:
       anchor_linenums: true
       line_spans: __span
       pygments_lang_class: true
-  - pymdownx.inlinehilite
+  - pymdownx.inlinehilite:
+      style_plain_text: shell
   - pymdownx.superfences
   - pymdownx.snippets
   - pymdownx.tabbed:
       alternate_style: true
-  - pymdownx.tasklist:
-      clickable_checkbox: true
   - toc:
       permalink: "#"
       toc_depth: 3

pyproject.toml

@@ -2,7 +2,7 @@
 minversion = "6.0"
 # opts:
 # `--strict-markers` - Raise error on unexpected pytest markers being used (add new markers to `markers` config)
-# `-n2` - parallelise over two threads (uses pytest-xdist)
+# `-nauto` - parallelise over as many threads are available (uses pytest-xdist). This auto-selects 1 thread when using the `--pdb` debugging flag.
 # `--nbmake --nbmake-kernel=pam` - test example notebooks using the "pam" notebook kernel (uses nbmake)
 # `--cov --cov-report=xml --cov-config=pyproject.toml` - generate coverage report for tests (uses pytest-cov; call `--no-cov` in CLI to switch off; `--cov-config` include to avoid bug)
 # `-m 'not high_mem'` - Do not run tests marked as consuming large amounts of memory (call `-m "high_mem"` in CLI to invert this; only `high_mem` marked tests will be run)

requirements/dev.txt

@@ -1,10 +1,10 @@
 cruft >= 2, < 3
 jupyter < 2
 mike >= 2, < 3
-mkdocs < 1.6
+mkdocs >= 1.6, < 2
 mkdocs-material >= 9.4, < 10
 mkdocs-click < 0.7
-mkdocs-jupyter < 0.24.7
+mkdocs-jupyter >= 0.24.8, < 0.25
 mkdocstrings-python < 2
 nbmake >= 1.5.1, < 2
 pre-commit < 4

src/pam/operations/cropping.py

@@ -18,23 +18,36 @@ def simplify_population(
 ) -> None:
     """Simplify external plans across a population."""
     # simplify plans
+
     for hid, pid, person in population.people():
         simplify_external_plans(person.plan, boundary, snap_to_boundary, rename_external_activities)
 
     # remove empty person-plans and households
     remove_persons = []
+    track_those_removed = []
+
+    print("Before simplification", population.stats)
+
     for hid, pid, person in population.people():
         if len(person.plan) == 1 and person.plan.day[0].act == "external":
             remove_persons.append((hid, pid))
+            track_those_removed.append({"hid": hid, "pid": pid})
     for hid, pid in remove_persons:
         del population[hid].people[pid]
 
+    print(len(remove_persons), "persons to be removed")
+
     remove_hhs = [
         hid for hid in population.households if len(population.households[hid].people) == 0
     ]
+
+    print(len(remove_hhs), "households to be removed")
+
     for hid in remove_hhs:
         del population.households[hid]
 
+    print("After simplification", population.stats)
+
 
 def simplify_external_plans(
     plan: Plan,