Side-by-side testing of Paver checks and New checks

.annotation_safe_list.yml

@@ -142,8 +142,6 @@ workflow.AssessmentWorkflowStep:
 # Via edx-celeryutils
 celery_utils.ChordData:
   ".. no_pii:": "No PII"
-celery_utils.FailedTask:
-  ".. no_pii:": "No PII"
 
 # Via completion XBlock
 completion.BlockCompletion:

.github/workflows/js-tests-paver.yml

@@ -0,0 +1,84 @@
+name: Javascript tests PAVER
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+
+jobs:
+  run_tests:
+    name: JS
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node-version: [18, 20]
+        python-version:
+          - "3.11"
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Fetch master to compare coverage
+        run: git fetch --depth=1 origin master
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Setup npm
+        run: npm i -g npm@10.5.x
+
+      - name: Install Firefox 123.0
+        run: |
+          sudo apt-get purge firefox
+          wget "https://ftp.mozilla.org/pub/firefox/releases/123.0/linux-x86_64/en-US/firefox-123.0.tar.bz2"
+          tar -xjf firefox-123.0.tar.bz2
+          sudo mv firefox /opt/firefox
+          sudo ln -s /opt/firefox/firefox /usr/bin/firefox
+
+      - name: Install Required System Packages
+        run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Get pip cache dir
+        id: pip-cache-dir
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Cache pip dependencies
+        id: cache-dependencies
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pip-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }}
+          restore-keys: ${{ runner.os }}-pip-
+
+      - name: Install Required Python Dependencies
+        run: |
+          make base-requirements
+
+      - uses: c-hive/gha-npm-cache@v1
+      - name: Run JS Tests
+        env:
+          TEST_SUITE: js-unit
+          SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
+        run: |
+          npm install -g jest
+          xvfb-run --auto-servernum ./scripts/all-tests.sh
+
+      - name: Save Job Artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: Build-Artifacts
+          path: |
+            reports/**/*
+            test_root/log/*.png
+            test_root/log/*.log
+            **/TEST-*.xml
+          overwrite: true

.github/workflows/js-tests.yml

@@ -64,13 +64,13 @@ jobs:
           make base-requirements
 
       - uses: c-hive/gha-npm-cache@v1
+
+      - name: Install npm
+        run: npm ci  
+
       - name: Run JS Tests
-        env:
-          TEST_SUITE: js-unit
-          SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
         run: |
-          npm install -g jest
-          xvfb-run --auto-servernum ./scripts/all-tests.sh
+          npm run test
 
       - name: Save Job Artifacts
         uses: actions/upload-artifact@v4

.github/workflows/quality-checks-paver.yml

@@ -0,0 +1,82 @@
+name: Quality checks PAVER
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - open-release/lilac.master
+
+jobs:
+  run_tests:
+    name: Quality Others
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+        python-version:
+          - "3.11"
+        node-version: [20]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Fetch base branch for comparison
+        run: git fetch --depth=1 origin ${{ github.base_ref }}
+
+      - name: Install Required System Packages
+        run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Setup npm
+        run: npm i -g npm@8.5.x
+
+      - name: Get pip cache dir
+        id: pip-cache-dir
+        run: |
+          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Cache pip dependencies
+        id: cache-dependencies
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.pip-cache-dir.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
+          restore-keys: ${{ runner.os }}-pip-
+
+      - name: Install Required Python Dependencies
+        env:
+          PIP_SRC: ${{ runner.temp }}
+        run: |
+          make test-requirements
+
+      - name: Run Quality Tests
+        env:
+          TEST_SUITE: quality
+          SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
+          PIP_SRC: ${{ runner.temp }}
+          TARGET_BRANCH: ${{ github.base_ref }}
+        run: |
+          ./scripts/all-tests.sh
+
+      - name: Save Job Artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Build-Artifacts
+          path: |
+            **/reports/**/*
+            test_root/log/**/*.log
+            *.log
+          overwrite: true

.github/workflows/quality-checks.yml

@@ -60,16 +60,29 @@ jobs:
           PIP_SRC: ${{ runner.temp }}
         run: |
           make test-requirements
-
+
+      - name: Install npm
+        env:
+          PIP_SRC: ${{ runner.temp }}
+        run: npm ci  
+
+      - name: Install python packages
+        env:
+          PIP_SRC: ${{ runner.temp }}
+        run: |
+            pip install -e .
+
       - name: Run Quality Tests
         env:
-          TEST_SUITE: quality
-          SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
           PIP_SRC: ${{ runner.temp }}
           TARGET_BRANCH: ${{ github.base_ref }}
         run: |
-          ./scripts/all-tests.sh
-
+            make pycodestyle
+            npm run lint
+            make xsslint
+            make pii_check
+            make check_keywords
+
       - name: Save Job Artifacts
         if: always()
         uses: actions/upload-artifact@v4

.pii_annotations.yml

@@ -1,7 +1,7 @@
 source_path: ./
 report_path: pii_report
 safelist_path: .annotation_safe_list.yml
-coverage_target: 94.5
+coverage_target: 83.3
 # See OEP-30 for more information on these values and what they mean:
 # https://open-edx-proposals.readthedocs.io/en/latest/oep-0030-arch-pii-markup-and-auditing.html#docstring-annotations
 annotations:

Makefile

@@ -204,3 +204,37 @@ migrate: migrate-lms migrate-cms
 # Part of https://github.com/openedx/wg-developer-experience/issues/136
 ubuntu-requirements: ## Install ubuntu 22.04 system packages needed for `pip install` to work on ubuntu.
 	sudo apt install libmysqlclient-dev libxmlsec1-dev
+
+xsslint: ## check xss for quality issuest
+	python scripts/xsslint/xss_linter.py \
+	--rule-totals \
+	--config=scripts.xsslint_config \
+	--thresholds=scripts/xsslint_thresholds.json
+
+pycodestyle: ## check python files for quality issues 
+	pycodestyle .
+
+## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved
+pii_check: ## check django models for pii annotations
+	DJANGO_SETTINGS_MODULE=cms.envs.test \
+	code_annotations django_find_annotations \
+		--config_file .pii_annotations.yml \
+		--app_name cms \
+		--coverage \
+		--lint
+
+	DJANGO_SETTINGS_MODULE=lms.envs.test \
+	code_annotations django_find_annotations \
+		--config_file .pii_annotations.yml \
+		--app_name lms \
+		--coverage \
+		--lint	
+
+check_keywords: ## check django models for reserve keywords
+	DJANGO_SETTINGS_MODULE=cms.envs.test \
+	python manage.py cms check_reserved_keywords \
+	--override_file db_keyword_overrides.yml
+
+	DJANGO_SETTINGS_MODULE=lms.envs.test \
+	python manage.py lms check_reserved_keywords \
+	--override_file db_keyword_overrides.yml

package.json

@@ -12,7 +12,26 @@
     "compile-sass-dev": "scripts/compile_sass.py --env=development",
     "watch": "{ npm run watch-webpack& npm run watch-sass& } && sleep infinity",
     "watch-webpack": "npm run webpack-dev -- --watch",
-    "watch-sass": "scripts/watch_sass.sh"
+    "watch-sass": "scripts/watch_sass.sh",
+    "lint": "python scripts/eslint.py",
+    "test": "npm run test-cms && npm run test-lms && npm run test-xmodule && npm run test-common && npm run test-jest",
+    "test-kind-vanilla": "npm run test-cms-vanilla && npm run test-xmodule-vanilla && npm run test-common-vanilla",
+    "test-kind-require": "npm run test-cms-require && npm run test-common-require",
+    "test-kind-webpack": "npm run test-cms-webpack && npm run test-lms-webpack && npm run test-xmodule-webpack",
+    "test-cms": "npm run test-cms-vanilla && npm run test-cms-require",
+    "test-cms-vanilla": "npm run test-suite -- cms/static/karma_cms.conf.js",
+    "test-cms-require": "npm run test-suite -- cms/static/karma_cms_squire.conf.js",
+    "test-cms-webpack": "npm run test-suite -- cms/static/karma_cms_webpack.conf.js",
+    "test-lms": "echo 'WARNING: Webpack JS tests are disabled. No LMS JS tests will be run. See https://github.com/openedx/edx-platform/issues/35956 for details.'",
+    "test-lms-webpack": "npm run test-suite -- lms/static/karma_lms.conf.js",
+    "test-xmodule": "npm run test-xmodule-vanilla",
+    "test-xmodule-vanilla": "npm run test-suite -- xmodule/js/karma_xmodule.conf.js",
+    "test-xmodule-webpack": "npm run test-suite -- xmodule/js/karma_xmodule_webpack.conf.js",
+    "test-common": "npm run test-common-vanilla && npm run test-common-require",
+    "test-common-vanilla": "npm run test-suite -- common/static/karma_common.conf.js",
+    "test-common-require": "npm run test-suite -- common/static/karma_common_requirejs.conf.js",
+    "test-suite": "${NODE_WRAPPER:-xvfb-run --auto-servernum} node --max_old_space_size=4096 node_modules/.bin/karma start --single-run=true --capture-timeout=60000 --browsers=FirefoxNoUpdates",
+    "test-jest": "jest"
   },
   "dependencies": {
     "@babel/core": "7.26.0",

pavelib/paver_tests/test_timer.py

@@ -77,9 +77,17 @@ def test_times(self):
         messages = self.get_log_messages()
         assert len(messages) == 1
 
-        assert 'duration' in messages[0] and messages[0]['duration'] == 35.6
-        assert 'started_at' in messages[0] and messages[0]['started_at'] == start.isoformat(' ')
-        assert 'ended_at' in messages[0] and messages[0]['ended_at'] == end.isoformat(' ')
+        # I'm not using assertDictContainsSubset because it is
+        # removed in python 3.2 (because the arguments were backwards)
+        # and it wasn't ever replaced by anything *headdesk*
+        assert 'duration' in messages[0]
+        assert 35.6 == messages[0]['duration']
+
+        assert 'started_at' in messages[0]
+        assert start.isoformat(' ') == messages[0]['started_at']
+
+        assert 'ended_at' in messages[0]
+        assert end.isoformat(' ') == messages[0]['ended_at']
 
     @patch.object(timer, 'PAVER_TIMER_LOG', None)
     def test_no_logs(self):
@@ -91,18 +99,28 @@ def test_arguments(self):
         messages = self.get_log_messages(args=(1, 'foo'), kwargs=dict(bar='baz'))
         assert len(messages) == 1
 
-        assert 'args' in messages[0] and messages[0]['args'] == [repr(1), repr('foo')]
-        assert 'kwargs' in messages[0] and messages[0]['kwargs'] == {'bar': repr('baz')}
+        # I'm not using assertDictContainsSubset because it is
+        # removed in python 3.2 (because the arguments were backwards)
+        # and it wasn't ever replaced by anything *headdesk*
+        assert 'args' in messages[0]
+        assert [repr(1), repr('foo')] == messages[0]['args']
+        assert 'kwargs' in messages[0]
+        assert {'bar': repr('baz')} == messages[0]['kwargs']
 
     @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log')
     def test_task_name(self):
         messages = self.get_log_messages()
         assert len(messages) == 1
 
-        assert 'task' in messages[0] and messages[0]['task'] == 'pavelib.paver_tests.test_timer.identity'
+        # I'm not using assertDictContainsSubset because it is
+        # removed in python 3.2 (because the arguments were backwards)
+        # and it wasn't ever replaced by anything *headdesk*
+        assert 'task' in messages[0]
+        assert 'pavelib.paver_tests.test_timer.identity' == messages[0]['task']
 
     @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log')
     def test_exceptions(self):
+
         @timer.timed
         def raises():
             """
@@ -113,7 +131,11 @@ def raises():
         messages = self.get_log_messages(task=raises, raises=Exception)
         assert len(messages) == 1
 
-        assert 'exception' in messages[0] and messages[0]['exception'] == 'Exception: The Message!'
+        # I'm not using assertDictContainsSubset because it is
+        # removed in python 3.2 (because the arguments were backwards)
+        # and it wasn't ever replaced by anything *headdesk*
+        assert 'exception' in messages[0]
+        assert 'Exception: The Message!' == messages[0]['exception']
 
     @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log-%Y-%m-%d-%H-%M-%S.log')
     def test_date_formatting(self):

requirements/edx/base.txt

@@ -504,7 +504,7 @@ edx-opaque-keys[django]==2.11.0
     #   ora2
 edx-organizations==6.13.0
     # via -r requirements/edx/kernel.in
-edx-proctoring==5.0.1
+edx-proctoring==4.18.4
     # via
     #   -r requirements/edx/kernel.in
     #   edx-proctoring-proctortrack

requirements/edx/development.txt

@@ -795,7 +795,7 @@ edx-organizations==6.13.0
     # via
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
-edx-proctoring==5.0.1
+edx-proctoring==4.18.4
     # via
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt