From fb45770fbc57df050c142e79edb910c1aaf87b51 Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Tue, 21 May 2024 09:47:27 -0600 Subject: [PATCH] fixing flags to customize how much work gets done when a library is loaded --- .pre-commit-config.yaml | 2 +- buildingmotif/dataclasses/library.py | 78 ++++++++++++++++++---- tests/integration/conftest.py | 4 +- tests/integration/test_library_validity.py | 63 ++++++++--------- 4 files changed, 95 insertions(+), 52 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8483923dc..6357a25c1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: entry: poetry run flake8 buildingmotif # can't poetry run becuase not present in repository https://github.com/pre-commit/mirrors-mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.10.0 hooks: - id: mypy args: ["--install-types", "--non-interactive", "--ignore-missing-imports", "--follow-imports=skip", "--disable-error-code=import-untyped"] diff --git a/buildingmotif/dataclasses/library.py b/buildingmotif/dataclasses/library.py index 8b16446ea..39af7dac8 100644 --- a/buildingmotif/dataclasses/library.py +++ b/buildingmotif/dataclasses/library.py @@ -141,6 +141,8 @@ def load( directory: Optional[str] = None, name: Optional[str] = None, overwrite: Optional[bool] = True, + infer_templates: Optional[bool] = True, + run_shacl_inference: Optional[bool] = True, ) -> "Library": """Loads a library from the database or an external source. When specifying a path to load a library or ontology_graph from, @@ -162,6 +164,12 @@ def load( :param overwrite: if true, replace any existing copy of the library, defaults to True :type overwrite: Optional[true], optional + :param infer_templates: if true, infer shapes from the ontology graph, + defaults to True + :type infer_templates: Optional[bool], optional + :param run_shacl_inference: if true, run SHACL inference on the ontology graph, + using the BuildingMOTIF SHACL engine, defaults to True + :type run_shacl_inference: Optional[bool], optional :return: the loaded library :rtype: Library :raises Exception: if the library cannot be loaded @@ -180,7 +188,12 @@ def load( ontology_graph.parse( ontology_graph_path, format=guess_format(ontology_graph_path) ) - return cls._load_from_ontology(ontology_graph, overwrite=overwrite) + return cls._load_from_ontology( + ontology_graph, + overwrite=overwrite, + infer_templates=infer_templates, + run_shacl_inference=run_shacl_inference, + ) elif directory is not None: if resource_exists("buildingmotif.libraries", directory): logging.debug(f"Loading builtin library: {directory}") @@ -191,7 +204,12 @@ def load( src = pathlib.Path(directory) if not src.exists(): raise Exception(f"Directory {src} does not exist") - return cls._load_from_directory(src, overwrite=overwrite) + return cls._load_from_directory( + src, + overwrite=overwrite, + infer_templates=infer_templates, + run_shacl_inference=run_shacl_inference, + ) elif name is not None: bm = get_building_motif() db_library = bm.table_connection.get_db_library_by_name(name) @@ -215,7 +233,11 @@ def _load_from_db(cls, id: int) -> "Library": @classmethod def _load_from_ontology( - cls, ontology: rdflib.Graph, overwrite: Optional[bool] = True + cls, + ontology: rdflib.Graph, + overwrite: Optional[bool] = True, + infer_templates: Optional[bool] = True, + run_shacl_inference: Optional[bool] = True, ) -> "Library": """ Load a library from an ontology graph. This proceeds as follows. @@ -229,6 +251,10 @@ def _load_from_ontology( :type ontology: rdflib.Graph :param overwrite: if true, overwrite the existing copy of the Library :type overwrite: bool + :param infer_templates: if true, infer shapes from the ontology graph + :type infer_templates: bool + :param run_shacl_inference: if true, run SHACL inference on the ontology graph + :type run_shacl_inference: bool :return: the loaded Library :rtype: "Library" """ @@ -236,7 +262,7 @@ def _load_from_ontology( # any=False will raise an error if there is more than one ontology defined in the graph ontology_name = ontology.value( predicate=rdflib.RDF.type, object=rdflib.OWL.Ontology, any=False - ) + ) or rdflib.URIRef("urn:unnamed/") if not overwrite: if cls._library_exists(ontology_name): @@ -248,12 +274,16 @@ def _load_from_ontology( # expand the ontology graph before we insert it into the database. This will ensure # that the output of compiled models will not contain triples that really belong to # the ontology - ontology = shacl_inference(ontology, engine=get_building_motif().shacl_engine) + if run_shacl_inference: + ontology = shacl_inference( + ontology, engine=get_building_motif().shacl_engine + ) lib = cls.create(ontology_name, overwrite=overwrite) - # infer shapes from any class/nodeshape candidates in the graph - lib._infer_shapes_from_graph(ontology) + if infer_templates: + # infer shapes from any class/nodeshape candidates in the graph + lib._infer_templates_from_graph(ontology) # load the ontology graph as a shape_collection shape_col_id = lib.get_shape_collection().id @@ -263,10 +293,10 @@ def _load_from_ontology( return lib - def _infer_shapes_from_graph(self, graph: rdflib.Graph): - """Infer shapes from a graph and add them to this library. + def _infer_templates_from_graph(self, graph: rdflib.Graph): + """Infer templates from a graph (by interpreting shapes) and add them to this library. - :param graph: graph to infer shapes from + :param graph: graph to infer templates from :type graph: rdflib.Graph """ class_candidates = set(graph.subjects(rdflib.RDF.type, rdflib.OWL.Class)) @@ -284,12 +314,21 @@ def _infer_shapes_from_graph(self, graph: rdflib.Graph): self._resolve_template_dependencies(template_id_lookup, dependency_cache) - def _load_shapes_from_directory(self, directory: pathlib.Path): + def _load_shapes_from_directory( + self, + directory: pathlib.Path, + infer_templates: Optional[bool] = True, + run_shacl_inference: Optional[bool] = True, + ): """Helper method to read all graphs in the given directory into this library. :param directory: directory containing graph files :type directory: pathlib.Path + :param infer_templates: if true, infer shapes from the ontology graph + :type infer_templates: bool + :param run_shacl_inference: if true, run SHACL inference on the ontology graph + :type run_shacl_inference: bool """ shape_col_id = self.get_shape_collection().id assert shape_col_id is not None # this should always pass @@ -302,12 +341,21 @@ def _load_shapes_from_directory(self, directory: pathlib.Path): f"Could not parse file {filename}: {e}" ) raise e + if run_shacl_inference: + shape_col.graph = shacl_inference( + shape_col.graph, engine=get_building_motif().shacl_engine + ) # infer shapes from any class/nodeshape candidates in the graph - self._infer_shapes_from_graph(shape_col.graph) + if infer_templates: + self._infer_templates_from_graph(shape_col.graph) @classmethod def _load_from_directory( - cls, directory: pathlib.Path, overwrite: Optional[bool] = True + cls, + directory: pathlib.Path, + overwrite: Optional[bool] = True, + infer_templates: Optional[bool] = True, + run_shacl_inference: Optional[bool] = True, ) -> "Library": """ Load a library from a directory. @@ -319,6 +367,10 @@ def _load_from_directory( :type directory: pathlib.Path :param overwrite: if true, overwrite the existing copy of the Library :type overwrite: bool + :param infer_templates: if true, infer shapes from the ontology graph + :type infer_templates: bool + :param run_shacl_inference: if true, run SHACL inference on the ontology graph + :type run_shacl_inference: bool :raises e: if cannot create template :raises e: if cannot resolve dependencies :return: library diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 446d8616f..b49c32f10 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -72,7 +72,9 @@ def pytest_generate_tests(metafunc): ): bm = BuildingMOTIF("sqlite://") - Library.load(ontology_graph="libraries/brick/Brick.ttl") + Library.load( + ontology_graph="libraries/brick/Brick.ttl", run_shacl_inference=False + ) templates = [] # load library for library_path in brick_libraries: diff --git a/tests/integration/test_library_validity.py b/tests/integration/test_library_validity.py index 8892f3885..334c967de 100644 --- a/tests/integration/test_library_validity.py +++ b/tests/integration/test_library_validity.py @@ -66,7 +66,11 @@ def test_223p_library(bm, library_path_223p: Path): @pytest.mark.integration def test_223p_template(bm, library_path_223p, template_223p, shacl_engine): bm.shacl_engine = shacl_engine - ont_223p = Library.load(ontology_graph="libraries/ashrae/223p/ontology/223p.ttl") + ont_223p = Library.load( + ontology_graph="libraries/ashrae/223p/ontology/223p.ttl", + infer_templates=False, + run_shacl_inference=False, + ) # pyshacl evaluation takes a long time, so we only test a couple of templates # from specific libraries @@ -80,7 +84,11 @@ def test_223p_template(bm, library_path_223p, template_223p, shacl_engine): if template_223p not in use_pyshacl[library_path_223p]: pytest.skip("pyshacl evaluation is slow, skipping this template") - lib = Library.load(directory=str(library_path_223p)) + lib = Library.load( + directory=str(library_path_223p), + infer_templates=False, + run_shacl_inference=False, + ) template_223p = lib.get_template_by_name(template_223p) @@ -105,43 +113,24 @@ def test_223p_template(bm, library_path_223p, template_223p, shacl_engine): @pytest.mark.integration def test_brick_template(bm, library_path_brick, template_brick, shacl_engine): bm.shacl_engine = shacl_engine - deps = [] - deps.append(Library.load(ontology_graph="libraries/brick/imports/ref-schema.ttl")) - deps.append( - Library.load( - ontology_graph="libraries/qudt/VOCAB_QUDT-QUANTITY-KINDS-ALL-v2.1.ttl" - ) - ) - deps.append( - Library.load( - ontology_graph="libraries/qudt/VOCAB_QUDT-DIMENSION-VECTORS-v2.1.ttl" - ) - ) - deps.append( - Library.load(ontology_graph="libraries/qudt/VOCAB_QUDT-UNITS-ALL-v2.1.ttl") - ) - deps.append( - Library.load(ontology_graph="libraries/qudt/SCHEMA-FACADE_QUDT-v2.1.ttl") - ) - deps.append( - Library.load(ontology_graph="libraries/qudt/SCHEMA_QUDT_NoOWL-v2.1.ttl") - ) - deps.append( - Library.load(ontology_graph="libraries/qudt/VOCAB_QUDT-PREFIXES-v2.1.ttl") - ) - deps.append( + dependency_graphs = [ + "libraries/brick/imports/ref-schema.ttl", + "libraries/qudt/VOCAB_QUDT-QUANTITY-KINDS-ALL-v2.1.ttl", + "libraries/qudt/VOCAB_QUDT-DIMENSION-VECTORS-v2.1.ttl", + "libraries/qudt/VOCAB_QUDT-UNITS-ALL-v2.1.ttl", + "libraries/qudt/SCHEMA-FACADE_QUDT-v2.1.ttl", + "libraries/qudt/SCHEMA_QUDT_NoOWL-v2.1.ttl", + "libraries/qudt/VOCAB_QUDT-PREFIXES-v2.1.ttl", + "libraries/qudt/SHACL-SCHEMA-SUPPLEMENT_QUDT-v2.1.ttl", + "libraries/qudt/VOCAB_QUDT-SYSTEM-OF-UNITS-ALL-v2.1.ttl", + "libraries/brick/imports/rec.ttl", + "libraries/brick/imports/recimports.ttl", + "libraries/brick/imports/brickpatches.ttl", + ] + for dep in dependency_graphs: Library.load( - ontology_graph="libraries/qudt/SHACL-SCHEMA-SUPPLEMENT_QUDT-v2.1.ttl" + ontology_graph=dep, infer_templates=False, run_shacl_inference=False ) - ) - deps.append( - Library.load( - ontology_graph="libraries/qudt/VOCAB_QUDT-SYSTEM-OF-UNITS-ALL-v2.1.ttl" - ) - ) - deps.append(Library.load(ontology_graph="libraries/brick/imports/rec.ttl")) - deps.append(Library.load(ontology_graph="libraries/brick/imports/recimports.ttl")) - deps.append(Library.load(ontology_graph="libraries/brick/imports/brickpatches.ttl")) ont_brick = Library.load(ontology_graph="libraries/brick/Brick.ttl") lib = Library.load(directory=str(library_path_brick))