updated build and dependencies (#77)

* march configurable in Dockerfile and used throughout

* build latest tcmalloc-minimal in Dockerfile

* simplified tests CMakeLists.txt

* updated sparql-parser-base to 0.2.2

* updated hypertrie and rdf-parser

CMakeLists.txt

@@ -75,6 +75,15 @@ find_package(optional-lite REQUIRED)
 # make a library of the code
 add_library(tentris INTERFACE)
 
+include(FetchContent)
+FetchContent_Declare(
+        cppitertools
+        GIT_REPOSITORY https://github.com/ryanhaining/cppitertools.git
+        GIT_TAG v2.1
+        GIT_SHALLOW TRUE)
+
+FetchContent_MakeAvailable(cppitertools)
+
 target_link_libraries(tentris
         INTERFACE
         stdc++fs # for #include <filesystem>
@@ -87,6 +96,7 @@ target_link_libraries(tentris
         rdf-parser::rdf-parser
         Boost::Boost
         serd-0
+        cppitertools::cppitertools
         )
 
 target_include_directories(tentris INTERFACE

Dockerfile

@@ -1,14 +1,30 @@
 FROM ubuntu:groovy AS builder
 ARG DEBIAN_FRONTEND=noninteractive
-ARG CXXFLAGS="${CXXFLAGS} -march=x86-64"
 ARG CMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld-11"
 ARG TENTRIS_MARCH="x86-64"
 
 RUN apt-get -qq update && \
-    apt-get -qq install -y make cmake uuid-dev git openjdk-11-jdk python3-pip python3-setuptools python3-wheel libstdc++-10-dev clang-11 g++-10 pkg-config google-perftools libgoogle-perftools-dev
+    apt-get -qq install -y make cmake uuid-dev git openjdk-11-jdk python3-pip python3-setuptools python3-wheel libstdc++-10-dev clang-11 g++-10 pkg-config
 
 ARG CXX="clang++-11"
 ARG CC="clang-11"
+ENV CXXFLAGS="${CXXFLAGS} -march=${TENTRIS_MARCH}"
+
+# Compile more recent tcmalloc-minimal with clang-11 + -march
+RUN git clone --quiet --branch gperftools-2.8.1 https://github.com/gperftools/gperftools
+WORKDIR /gperftools/build
+RUN cmake \
+    -Dgperftools_build_minimal=ON \
+    -DGPERFTOOLS_BUILD_DEBUGALLOC=OFF \
+    -Dgperftools_build_benchmark=OFF \
+    -Dgperftools_enable_libunwind=ON \
+    -gperftools_dynamic_sized_delete_support=ON \
+    -gperftools_sized_delete=ON \
+    .. && \
+    make -j && \
+    make install
+WORKDIR /
+
 # we need serd as static library. Not available from ubuntu repos
 RUN ln -s /usr/bin/python3 /usr/bin/python
 RUN git clone --quiet --branch v0.30.8 https://gitlab.com/drobilla/serd.git
@@ -23,7 +39,12 @@ WORKDIR /
 RUN pip3 install conan && \
     conan user && \
     conan profile new --detect default && \
-    conan profile update settings.compiler.libcxx=libstdc++11 default
+    conan profile update settings.compiler.libcxx=libstdc++11 default && \
+    conan profile update env.CXXFLAGS="${CXXFLAGS}" default && \
+    conan profile update env.CMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" default && \
+    conan profile update env.CXX="${CXX}" default && \
+    conan profile update env.CC="${CC}" default && \
+    conan profile update options.boost:extra_b2_flags="cxxflags=\\\"${CXXFLAGS}\\\"" default
 
 # add conan repositories
 RUN conan remote add tsl https://api.bintray.com/conan/tessil/tsl
@@ -50,7 +71,6 @@ RUN cmake -DCMAKE_BUILD_TYPE=Release -DTENTRIS_BUILD_WITH_TCMALLOC=true -DTENTRI
 RUN make -j $(nproc)
 
 FROM scratch
-WORKDIR /tentris
 COPY --from=builder /tentris/build/tentris_server /tentris_server
 COPY --from=builder /tentris/build/tentris_terminal /tentris_terminal
 COPY --from=builder /tentris/build/ids2hypertrie /ids2hypertrie

conanfile.txt

@@ -1,11 +1,10 @@
 [requires]
 boost/1.75.0
 fmt/7.1.3
-abseil/20200225.2 # remove when rdf-parser is updated
 restinio/0.6.12
-hypertrie/0.6.0-rc13@dice-group/stable
-rdf-parser/0.10@dice-group/stable
-sparql-parser-base/0.2.1@dice-group/stable
+hypertrie/0.6.0@dice-group/rc14
+rdf-parser/0.13.0@dice-group/stable
+sparql-parser-base/0.2.2@dice-group/stable
 
 [options]
 restinio:asio=boost

src/exec/TentrisTerminal.cpp

@@ -20,16 +20,16 @@
 #include <itertools.hpp>
 #include "tentris/http/QueryResultState.hpp"
 
-namespace {
-	using namespace tentris::store;
-	using namespace tentris::logging;
-	using namespace tentris::store::cache;
-	using namespace tentris::store::sparql;
-	using namespace std::filesystem;
-	using namespace iter;
-	using namespace tentris::tensor;
-	using namespace std::chrono;
-}
+using namespace tentris::store;
+using namespace tentris::logging;
+using namespace tentris::store::cache;
+using namespace tentris::store::sparql;
+using namespace std::filesystem;
+using namespace iter;
+using namespace tentris::tensor;
+using namespace std::chrono;
+
+using Variable = Dice::sparql::Variable;
 
 TerminalConfig cfg;
 

src/exec/tools/IDs2Hypertrie.cpp

@@ -38,7 +38,6 @@ int main(int argc, char *argv[]) {
 }
 
 namespace tentris::IDs2Hypertrie {
-	using namespace rdf_parser::Turtle;
 	using namespace tentris::store;
 	using namespace fmt::literals;
 	using namespace std::chrono;

src/exec/tools/RDF2IDs.cpp

@@ -6,10 +6,10 @@
 #include <tentris/store/RDF/TermStore.hpp>
 
 int main(int argc, char *argv[]) {
-	using namespace rdf_parser::Turtle;
 	using namespace tentris::store;
 	using namespace fmt::literals;
 	using namespace std::chrono;
+	using Triple = Dice::rdf::Triple;
 
 	if (argc != 2) {
 		std::cerr << "Please provide exactly one triple file as commandline argument." << std::endl;

src/lib/tentris/http/SparqlEndpoint.hpp

@@ -42,6 +42,16 @@ namespace tentris::http {
 		template<typename output_type_t> requires std::is_same_v<output_type_t, restinio::chunked_output_t> or
 												  std::is_same_v<output_type_t, restinio::restinio_controlled_output_t>
 		struct SparqlEndpoint {
+		private:
+			using Term = Dice::rdf::Term;
+			using BNode = Dice::rdf::BNode;
+			using Literal = Dice::rdf::Literal;
+			using URIRef = Dice::rdf::URIRef;
+			using Triple = Dice::rdf::Triple;
+			using TriplePattern = Dice::sparql::TriplePattern;
+			using Variable = Dice::sparql::Variable;
+		public:
+
 			constexpr static bool chunked_output = std::is_same_v<output_type_t, restinio::chunked_output_t>;
 			constexpr static size_t chunk_size = 100'000'000UL;
 

src/lib/tentris/store/QueryExecutionPackage.hpp

@@ -24,7 +24,7 @@ namespace tentris::store::cache {
 		using const_BoolHypertrie = ::tentris::tensor::const_BoolHypertrie;
 		using time_point_t = logging::time_point_t;
 		using SelectModifier = sparql::SelectModifier;
-		using Variable = sparql::Variable;
+		using Variable = Dice::sparql::Variable;
 		using ParsedSPARQL = sparql::ParsedSPARQL;
 		using Subscript = ::tentris::tensor::Subscript;
 

src/lib/tentris/store/RDF/SerdParser.hpp

@@ -3,8 +3,8 @@
 
 #include <fmt/format.h>
 #include <fmt/core.h>
-#include <Dice/rdf_parser/RDF/Triple.hpp>
-#include <Dice/rdf_parser/RDF/Term.hpp>
+#include <Dice/RDF/Triple.hpp>
+#include <Dice/RDF/Term.hpp>
 #include <Dice/hash/DiceHash.hpp>
 #include <boost/lockfree/spsc_queue.hpp>
 #include <serd-0/serd/serd.h>
@@ -18,7 +18,11 @@
 namespace tentris::store::rdf {
 
 	class BulkLoad {
-		using Triple = rdf_parser::store::rdf::Triple;
+		using Triple = Dice::rdf::Triple;
+		using Term = Dice::rdf::Term;
+		using BNode = Dice::rdf::BNode;
+		using Literal = Dice::rdf::Literal;
+		using URIRef = Dice::rdf::URIRef;
 		using prefixes_map_type = tsl::hopscotch_map<std::string, std::string, Dice::hash::DiceHash<std::string>>;
 		prefixes_map_type prefixes{};
 
@@ -157,8 +161,12 @@ namespace tentris::store::rdf {
 	};
 
 	class SerdParser {
+		using Triple = Dice::rdf::Triple;
+		using Term = Dice::rdf::Term;
+		using BNode = Dice::rdf::BNode;
+		using Literal = Dice::rdf::Literal;
+		using URIRef = Dice::rdf::URIRef;
 
-	private:
 		std::string file_name_;
 
 	public:

src/lib/tentris/store/RDF/TermStore.hpp

@@ -5,41 +5,39 @@
 #include <memory>
 #include <tuple>
 
-#include <Dice/rdf_parser/RDF/Term.hpp>
+#include <Dice/RDF/Term.hpp>
 
 #include <Dice/hash/DiceHash.hpp>
 
-template<>
-std::size_t Dice::hash::dice_hash(rdf_parser::store::rdf::Term const &term) noexcept {
-	return ::Dice::hash::dice_hash(term.getIdentifier());
-}
-
 namespace tentris::store::rdf {
 
 
 	/**
-	 * Replacement for std::hash based TermHash defined alongside with Term.
+	 * A hash for Terms that returns for Term* the hash hash(Term) instead of hash(Term*).
 	 */
 	struct TermHash {
-		size_t operator()(const rdf_parser::store::rdf::Term &term) const {
+	private:
+		using Term = Dice::rdf::Term;
+	public:
+		size_t operator()(Term const  &term) const {
 			return ::Dice::hash::dice_hash(term);
 		}
 
-		size_t operator()(const std::unique_ptr<rdf_parser::store::rdf::Term> &term_ptr) const {
+		size_t operator()(std::unique_ptr<Term> const &term_ptr) const {
 			return ::Dice::hash::dice_hash(*term_ptr);
 		}
 
-		size_t operator()(const rdf_parser::store::rdf::Term *&term_ptr) const {
+		size_t operator()(Term const *const term_ptr) const {
 			return ::Dice::hash::dice_hash(*term_ptr);
 		}
 	};
 
 
 	class TermStore {
-		using Term = rdf_parser::store::rdf::Term;
-		using BNode = rdf_parser::store::rdf::BNode;
-		using Literal = rdf_parser::store::rdf::Literal;
-		using URIRef = rdf_parser::store::rdf::URIRef;
+		using Term = Dice::rdf::Term;
+		using BNode = Dice::rdf::BNode;
+		using Literal = Dice::rdf::Literal;
+		using URIRef = Dice::rdf::URIRef;
 	public:
 		using set_type = tsl::sparse_set<std::unique_ptr<Term>,
 				TermHash,

src/lib/tentris/store/SPARQL/ParsedSPARQL.hpp

@@ -22,9 +22,10 @@
 #include <Dice/einsum/internal/Subscript.hpp>
 #include <utility>
 
-#include <Dice/rdf_parser/RDF/Term.hpp>
-#include "tentris/store/SPARQL/Variable.hpp"
-#include "tentris/store/SPARQL/TriplePattern.hpp"
+#include <Dice/RDF/Term.hpp>
+#include <Dice/RDF/ParseTerm.hpp>
+#include <Dice/RDF/Triple.hpp>
+#include <Dice/SPARQL/TriplePattern.hpp>
 
 
 namespace tentris::store::sparql {
@@ -44,10 +45,7 @@ namespace tentris::store::sparql {
 	};
 
 	class LexerErrorListener : public antlr4::BaseErrorListener {
-		using Term = rdf_parser::store::rdf::Term;
-		using BNode = rdf_parser::store::rdf::BNode;
-		using Literal = rdf_parser::store::rdf::Literal;
-		using URIRef = rdf_parser::store::rdf::URIRef;
+		using Variable = Dice::sparql::Variable;
 	public:
 		LexerErrorListener() = default;
 
@@ -73,6 +71,14 @@ namespace tentris::store::sparql {
 
 
 	class ParsedSPARQL {
+		using Term = Dice::rdf::Term;
+		using BNode = Dice::rdf::BNode;
+		using Literal = Dice::rdf::Literal;
+		using URIRef = Dice::rdf::URIRef;
+		using TriplePattern = Dice::sparql::TriplePattern;
+		using VarOrTerm = Dice::sparql::VarOrTerm;
+		using Variable = Dice::sparql::Variable;
+
 		using SparqlLexer = parser::SparqlLexer;
 		using ANTLRInputStream =antlr4::ANTLRInputStream;
 		using CommonTokenStream = antlr4::CommonTokenStream;
@@ -231,7 +237,7 @@ namespace tentris::store::sparql {
 			if (std::holds_alternative<Variable>(variant)) {
 				auto &var = std::get<Variable>(variant);
 				variables.insert(var);
-				if (var.is_anonym)
+				if (var.isAnon())
 					anonym_variables.insert(var);
 			}
 		}
@@ -307,7 +313,7 @@ namespace tentris::store::sparql {
 				else
 					return Variable{"__:" + std::to_string(next_anon_var_id++), true};
 			} else if (not termContext->NIL()) {
-				return Literal::make_term(termContext->getText());
+				return Dice::rdf::parse_term(termContext->getText());
 			} else {
 				throw std::logic_error{"Handling NIL not yet implemented."};
 				// TODO: handle NIL value "( )"