From ebc376a060319712e500aa260fb63f068b0e232e Mon Sep 17 00:00:00 2001 From: Dumitru Date: Tue, 7 Jan 2025 14:06:26 +0200 Subject: [PATCH 1/8] add failing tests for notice linkage in template --- tests/unit/notice_packager/conftest.py | 5 +++ .../test_template_generator.py | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/tests/unit/notice_packager/conftest.py b/tests/unit/notice_packager/conftest.py index 6ad49bb8..58113afb 100644 --- a/tests/unit/notice_packager/conftest.py +++ b/tests/unit/notice_packager/conftest.py @@ -98,3 +98,8 @@ def invalid_rdf_files_path(): @pytest.fixture def notice_id(): return "196390_2018" + + +@pytest.fixture +def work_id_predicate(): + return "http://publications.europa.eu/ontology/cdm#work_id" \ No newline at end of file diff --git a/tests/unit/notice_packager/test_template_generator.py b/tests/unit/notice_packager/test_template_generator.py index 330b63fd..b9f6626d 100644 --- a/tests/unit/notice_packager/test_template_generator.py +++ b/tests/unit/notice_packager/test_template_generator.py @@ -9,8 +9,10 @@ import re import pytest +from rdflib import Graph, Literal, XSD from ted_sws.notice_packager.adapters.template_generator import TemplateGenerator +from ted_sws.notice_packager.model.metadata import PackagerMetadata from tests import TEST_DATA_PATH @@ -57,3 +59,40 @@ def test_mets2action_mets_xml_generator_with_wrong_action(template_sample_metada template_sample_metadata.mets.type = "wrong_action" with pytest.raises(ValueError): TemplateGenerator.mets2action_mets_xml_generator(template_sample_metadata) + + +def test_mets_dmd_rdf_has_work_id_after_generation(template_sample_metadata: PackagerMetadata, + work_id_predicate: str): + mets_dmd_rdf: str = TemplateGenerator.mets_xml_dmd_rdf_generator(template_sample_metadata) + mets_graph: Graph = Graph().parse(data=mets_dmd_rdf, format="xml") + + work_id_predicate_exists: bool = mets_graph.query( + f""" ASK WHERE {{ ?subject <{work_id_predicate}> ?object . }} """).askAnswer + + assert work_id_predicate_exists + +def test_mets_dmd_rdf_has_work_id_as_string_after_generation(template_sample_metadata: PackagerMetadata, + work_id_predicate: str): + mets_dmd_rdf: str = TemplateGenerator.mets_xml_dmd_rdf_generator(template_sample_metadata) + mets_graph: Graph = Graph().parse(data=mets_dmd_rdf, format="xml") + string_datatype = XSD.string + + work_id_predicate_exists: bool = mets_graph.query( + f""" ASK WHERE {{ ?subject <{work_id_predicate}> ?object . FILTER(datatype(?object) = <{string_datatype}>) }} """).askAnswer + + assert work_id_predicate_exists + +def test_mets_dmd_rdf_has_correct_work_id_value_after_generation(template_sample_metadata: PackagerMetadata, + work_id_predicate: str): + mets_dmd_rdf: str = TemplateGenerator.mets_xml_dmd_rdf_generator(template_sample_metadata) + mets_graph: Graph = Graph().parse(data=mets_dmd_rdf, format="xml") + + assert template_sample_metadata.work.uri + work_id_value_literal = Literal(template_sample_metadata.work.uri, datatype=XSD.string) + + work_id_is_same: bool = mets_graph.query( + f""" ASK WHERE {{ ?subject <{work_id_predicate}> {work_id_value_literal.n3()} . }} """).askAnswer + + assert work_id_is_same + + From 99e30759b4bc2d4cf0ed0cd3b3e84c26e206c469 Mon Sep 17 00:00:00 2001 From: Dumitru Date: Tue, 7 Jan 2025 14:41:20 +0200 Subject: [PATCH 2/8] add work_id as datatype property pointing to notice graph uri --- .../notice_packager/resources/templates/mets_xml_dmd_rdf.jinja2 | 1 + 1 file changed, 1 insertion(+) diff --git a/ted_sws/notice_packager/resources/templates/mets_xml_dmd_rdf.jinja2 b/ted_sws/notice_packager/resources/templates/mets_xml_dmd_rdf.jinja2 index 775d1467..2fd521df 100644 --- a/ted_sws/notice_packager/resources/templates/mets_xml_dmd_rdf.jinja2 +++ b/ted_sws/notice_packager/resources/templates/mets_xml_dmd_rdf.jinja2 @@ -12,6 +12,7 @@ {# #} + {{ work.uri }} ted:{{ work.identifier }} oj:{{ work.oj_identifier }} From 780a3b2fa79ba0230b933ef35474a5d060d84b27 Mon Sep 17 00:00:00 2001 From: Dumitru Date: Wed, 8 Jan 2025 18:14:28 +0200 Subject: [PATCH 3/8] solve failing test comparing mets after generating --- .../templates/2021_S_004_003545_0.mets.xml.dmd.rdf | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_data/notice_packager/templates/2021_S_004_003545_0.mets.xml.dmd.rdf b/tests/test_data/notice_packager/templates/2021_S_004_003545_0.mets.xml.dmd.rdf index 7a4226f3..28562c5f 100644 --- a/tests/test_data/notice_packager/templates/2021_S_004_003545_0.mets.xml.dmd.rdf +++ b/tests/test_data/notice_packager/templates/2021_S_004_003545_0.mets.xml.dmd.rdf @@ -12,6 +12,7 @@ + http://data.europa.eu/a4g/resource/2021/003545_2021 ted:2021_S_004_003545 oj:JOS_2021_004_R_003545 From d217d2b066efc29e3e7f2c77d49a1e2734e868a8 Mon Sep 17 00:00:00 2001 From: Dumitru Date: Tue, 14 Jan 2025 10:43:35 +0200 Subject: [PATCH 4/8] Update __init__.py --- tests/unit/dags/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/dags/__init__.py b/tests/unit/dags/__init__.py index 1a2507f7..1e677c66 100644 --- a/tests/unit/dags/__init__.py +++ b/tests/unit/dags/__init__.py @@ -40,3 +40,4 @@ FULL_BRANCH_TASK_IDS = [START_PROCESSING_NOTICE_TASK_ID, INDEX_NOTICE_XML_CONTENT_TASK_ID, NORMALISE_NOTICE_METADATA_TASK_ID] + TRANSFORM_BRANCH_TASK_IDS + PACKAGE_BRANCH_TASK_IDS + PUBLISH_BRANCH_TASK_IDS + From a79de37f86a51637561451915dc5c2877074f2e4 Mon Sep 17 00:00:00 2001 From: Dragos0000 Date: Tue, 14 Jan 2025 09:16:31 +0000 Subject: [PATCH 5/8] testing sonar cube --- .github/workflows/unit-tests-srv.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests-srv.yml b/.github/workflows/unit-tests-srv.yml index 617bd7bc..84784070 100644 --- a/.github/workflows/unit-tests-srv.yml +++ b/.github/workflows/unit-tests-srv.yml @@ -52,7 +52,7 @@ jobs: name: codecov-umbrella fail_ci_if_error: true - name: SonarCloud Scan - uses: SonarSource/sonarcloud-github-action@master + uses: SonarSource/sonarqube-scan-action@v4.1.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} From ba852f8851f4e7229d83ba4b99974da2a83b5ef0 Mon Sep 17 00:00:00 2001 From: Dragos0000 Date: Tue, 14 Jan 2025 09:39:19 +0000 Subject: [PATCH 6/8] testing sonar cube --- .github/workflows/unit-tests-srv.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unit-tests-srv.yml b/.github/workflows/unit-tests-srv.yml index 84784070..306eff6a 100644 --- a/.github/workflows/unit-tests-srv.yml +++ b/.github/workflows/unit-tests-srv.yml @@ -51,6 +51,9 @@ jobs: env_vars: OS,PYTHON name: codecov-umbrella fail_ci_if_error: true + - name: Set working directory fir SonarCloud + run: mkdir -p $GITHUB_WORKSPACE/.scannerwork && chmod -R 777 $GITHUB_WORKSPACE/.scannerwork + - name: SonarCloud Scan uses: SonarSource/sonarqube-scan-action@v4.1.0 env: From be49ec08b264979f665474a170a89bc384fc0f01 Mon Sep 17 00:00:00 2001 From: Dragos0000 Date: Tue, 14 Jan 2025 10:37:01 +0000 Subject: [PATCH 7/8] testing sonar cube working dir --- .github/workflows/unit-tests-srv.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests-srv.yml b/.github/workflows/unit-tests-srv.yml index 306eff6a..48b951b0 100644 --- a/.github/workflows/unit-tests-srv.yml +++ b/.github/workflows/unit-tests-srv.yml @@ -52,13 +52,15 @@ jobs: name: codecov-umbrella fail_ci_if_error: true - name: Set working directory fir SonarCloud - run: mkdir -p $GITHUB_WORKSPACE/.scannerwork && chmod -R 777 $GITHUB_WORKSPACE/.scannerwork + run: mkdir -p $GITHUB_WORKSPACE/.scannerwork && chmod -R 777 $GITHUB_WORKSPACE/.scannerwork && pwd && ls -a - name: SonarCloud Scan uses: SonarSource/sonarqube-scan-action@v4.1.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + with: + args: -Dsonar.working.directory=$GITHUB_WORKSPACE/.scannerwork # - name: Clean Mongo DB # run: make clean-mongo-db # - name: Upload coverage to Codecov From 478bb69b00165657a601f59764fdeab81d5b6a46 Mon Sep 17 00:00:00 2001 From: Dumitru Date: Wed, 15 Jan 2025 07:40:50 +0200 Subject: [PATCH 8/8] add documentation --- tests/unit/notice_packager/conftest.py | 1 + tests/unit/notice_packager/test_template_generator.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/unit/notice_packager/conftest.py b/tests/unit/notice_packager/conftest.py index 58113afb..fb3a29f5 100644 --- a/tests/unit/notice_packager/conftest.py +++ b/tests/unit/notice_packager/conftest.py @@ -102,4 +102,5 @@ def notice_id(): @pytest.fixture def work_id_predicate(): + """Returns the URI predicate for the CDM work identifier.""" return "http://publications.europa.eu/ontology/cdm#work_id" \ No newline at end of file diff --git a/tests/unit/notice_packager/test_template_generator.py b/tests/unit/notice_packager/test_template_generator.py index b9f6626d..ad1456de 100644 --- a/tests/unit/notice_packager/test_template_generator.py +++ b/tests/unit/notice_packager/test_template_generator.py @@ -63,6 +63,7 @@ def test_mets2action_mets_xml_generator_with_wrong_action(template_sample_metada def test_mets_dmd_rdf_has_work_id_after_generation(template_sample_metadata: PackagerMetadata, work_id_predicate: str): + """Test that generated METS DMD RDF contains a work_id predicate.""" mets_dmd_rdf: str = TemplateGenerator.mets_xml_dmd_rdf_generator(template_sample_metadata) mets_graph: Graph = Graph().parse(data=mets_dmd_rdf, format="xml") @@ -71,8 +72,10 @@ def test_mets_dmd_rdf_has_work_id_after_generation(template_sample_metadata: Pac assert work_id_predicate_exists + def test_mets_dmd_rdf_has_work_id_as_string_after_generation(template_sample_metadata: PackagerMetadata, - work_id_predicate: str): + work_id_predicate: str): + """Test that work_id in METS DMD RDF is of type xsd:string.""" mets_dmd_rdf: str = TemplateGenerator.mets_xml_dmd_rdf_generator(template_sample_metadata) mets_graph: Graph = Graph().parse(data=mets_dmd_rdf, format="xml") string_datatype = XSD.string @@ -82,8 +85,10 @@ def test_mets_dmd_rdf_has_work_id_as_string_after_generation(template_sample_met assert work_id_predicate_exists + def test_mets_dmd_rdf_has_correct_work_id_value_after_generation(template_sample_metadata: PackagerMetadata, work_id_predicate: str): + """Test that work_id value in METS DMD RDF matches the metadata work URI.""" mets_dmd_rdf: str = TemplateGenerator.mets_xml_dmd_rdf_generator(template_sample_metadata) mets_graph: Graph = Graph().parse(data=mets_dmd_rdf, format="xml") @@ -94,5 +99,3 @@ def test_mets_dmd_rdf_has_correct_work_id_value_after_generation(template_sample f""" ASK WHERE {{ ?subject <{work_id_predicate}> {work_id_value_literal.n3()} . }} """).askAnswer assert work_id_is_same - -