tests on virtuoso

.github/workflows/run_tests.yml

@@ -38,11 +38,12 @@ jobs:
       run: |
         pip3 install poetry
         poetry install
-    - name: Run Blazegraph
+    - name: Run Virtuoso
       run: |
-        wget https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_2_1_6_RC/blazegraph.jar
-        java -server -jar blazegraph.jar &
-        python3 -c 'import time; time.sleep(15)'
+        cd virtuoso-opensource/bin
+        ./virtuoso-t
+        python3 -c 'import time; time.sleep(30)'
+        cd ../..
     - name: Check the overage
       run: |
         poetry run coverage run --rcfile=oc_ocdm/test/coverage/.coveragerc

oc_ocdm/test/reader/test_reader.py

@@ -27,7 +27,7 @@
 class TestReader(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        cls.endpoint = 'http://127.0.0.1:9999/blazegraph/sparql'
+        cls.endpoint = 'http://127.0.0.1:8804/sparql'
         BASE = os.path.join('oc_ocdm', 'test', 'reader')
         server = SPARQLWrapper(cls.endpoint)
         query = 'LOAD <file:' + os.path.abspath(os.path.join(BASE, f'br.nt')).replace('\\', '/') + '> INTO GRAPH <' + f'https://w3id.org/oc/meta/' + '>'

oc_ocdm/test/storer/test_storer.py

@@ -29,17 +29,28 @@
 
 from shutil import rmtree
 
+
 class TestStorer(unittest.TestCase):
+    ts = 'http://127.0.0.1:8804/sparql'
+
+    def reset_server(self, server:str=ts) -> None:
+        ts = SPARQLWrapper(server)
+        for graph in {'https://w3id.org/oc/meta/br/', 'https://w3id.org/oc/meta/ra/', 'https://w3id.org/oc/meta/re/', 'https://w3id.org/oc/meta/id/', 'https://w3id.org/oc/meta/ar/', 'http://default.graph/'}:
+            ts.setQuery(f'CLEAR GRAPH <{graph}>')
+            ts.setMethod(POST)
+            ts.query()
+
     def setUp(self):
         self.resp_agent = "http://resp_agent.test/"
         self.base_iri = "http://test/"
-        self.ts = 'http://127.0.0.1:9999/blazegraph/sparql'
+        self.ts = self.ts
         self.graph_set = GraphSet(self.base_iri, "", "060", False)
         self.prov_set = ProvSet(self.graph_set, self.base_iri, "", False)
         self.br = self.graph_set.add_br(self.resp_agent)
         self.data_dir = os.path.join("oc_ocdm", "test", "storer", "data")
         self.prov_dir = os.path.join("oc_ocdm", "test", "storer", "test_provenance")
         self.info_dir = os.path.join(self.prov_dir, "info_dir")
+        self.reset_server()
 
     def tearDown(self):
         if os.path.exists(self.data_dir):
@@ -161,10 +172,6 @@ def test_store_graphs_in_file_multiprocessing(self):
                 'http://www.w3.org/ns/prov#wasAttributedTo': [{'@id': 'http://resp_agent.test/'}]}], '@id': 'http://test/br/0601/prov/'}])
 
     def test_provenance(self):
-        ts = SPARQLWrapper(self.ts)
-        ts.setQuery('delete{?x ?y ?z} where{?x ?y ?z}')
-        ts.setMethod(POST)
-        ts.query()
         graph_set = GraphSet(self.base_iri, "", "060", False)
         prov_set = ProvSet(graph_set, self.base_iri, info_dir=self.info_dir)
         base_dir = os.path.join("oc_ocdm", "test", "storer", "test_provenance") + os.sep
@@ -207,7 +214,7 @@ def test_store_graphs_save_queries(self):
 
 def process_entity(entity):
     base_iri = "http://test/"
-    ts = 'http://127.0.0.1:9999/blazegraph/sparql'
+    ts = 'http://127.0.0.1:8804/sparql'
     resp_agent = "http://resp_agent.test/"
     base_dir = os.path.join("oc_ocdm", "test", "storer", "test_provenance") + os.sep
     info_dir = os.path.join("oc_ocdm", "test", "storer", "test_provenance", "info_dir")

virtuoso-opensource/bin/virtuoso.ini

@@ -0,0 +1,265 @@
+;
+;  virtuoso.ini
+;
+;  Configuration file for the OpenLink Virtuoso VDBMS Server
+;
+;  To learn more about this product, or any other product in our
+;  portfolio, please check out our web site at:
+;
+;      http://virtuoso.openlinksw.com/
+;
+;  or contact us at:
+;
+;      general.information@openlinksw.com
+;
+;  If you have any technical questions, please contact our support
+;  staff at:
+;
+;      technical.support@openlinksw.com
+;
+
+;
+;  Database setup
+;
+[Database]
+DatabaseFile			= ../database/virtuoso.db
+ErrorLogFile			= ../database/virtuoso.log
+LockFile			= ../database/virtuoso.lck
+TransactionFile			= ../database/virtuoso.trx
+xa_persistent_file		= ../database/virtuoso.pxa
+ErrorLogLevel			= 7
+FileExtend			= 200
+MaxCheckpointRemap		= 2000
+Striping			= 0
+TempStorage			= TempDatabase
+
+
+[TempDatabase]
+DatabaseFile			= ../database/virtuoso-temp.db
+TransactionFile			= ../database/virtuoso-temp.trx
+MaxCheckpointRemap		= 2000
+Striping			= 0
+
+
+;
+;  Server parameters
+;
+[Parameters]
+ServerPort			= 1104
+LiteMode			= 0
+DisableUnixSocket		= 1
+DisableTcpSocket		= 0
+;SSLServerPort			= 2111
+;SSLCertificate			= cert.pem
+;SSLPrivateKey			= pk.pem
+;X509ClientVerify		= 0
+;X509ClientVerifyDepth		= 0
+;X509ClientVerifyCAFile		= ca.pem
+MaxClientConnections		= 10
+CheckpointInterval		= 60
+O_DIRECT			= 0
+CaseMode			= 2
+MaxStaticCursorRows		= 5000
+CheckpointAuditTrail		= 0
+AllowOSCalls			= 0
+SchedulerInterval		= 10
+DirsAllowed			= ., ../vad, /usr/share/proj, ../../test
+ThreadCleanupInterval		= 1
+ThreadThreshold			= 10
+ResourcesCleanupInterval	= 1
+FreeTextBatchSize		= 100000
+SingleCPU			= 0
+VADInstallDir			= ../vad/
+PrefixResultNames               = 0
+RdfFreeTextRulesSize		= 100
+IndexTreeMaps			= 256
+MaxMemPoolSize                  = 200000000
+PrefixResultNames               = 0
+MacSpotlight                    = 0
+MaxQueryMem 		 	= 2G		; memory allocated to query processor
+VectorSize 		 	= 1000		; initial parallel query vector (array of query operations) size
+MaxVectorSize 		 	= 1000000	; query vector size threshold.
+AdjustVectorSize 	 	= 0
+ThreadsPerQuery 	 	= 4
+AsyncQueueMaxThreads 	 	= 10
+;;
+;; When running with large data sets, one should configure the Virtuoso
+;; process to use between 2/3 to 3/5 of free system memory and to stripe
+;; storage on all available disks.
+;;
+;; Uncomment next two lines if there is 2 GB system memory free
+; NumberOfBuffers          = 170000
+; MaxDirtyBuffers          = 130000
+;; Uncomment next two lines if there is 4 GB system memory free
+; NumberOfBuffers          = 340000
+; MaxDirtyBuffers          = 250000
+;; Uncomment next two lines if there is 8 GB system memory free
+; NumberOfBuffers          = 680000
+; MaxDirtyBuffers          = 500000
+;; Uncomment next two lines if there is 16 GB system memory free
+;NumberOfBuffers          = 1360000
+;MaxDirtyBuffers          = 1000000
+;; Uncomment next two lines if there is 32 GB system memory free
+;NumberOfBuffers          = 2720000
+;MaxDirtyBuffers          = 2000000
+;; Uncomment next two lines if there is 48 GB system memory free
+;NumberOfBuffers          = 4000000
+;MaxDirtyBuffers          = 3000000
+;; Uncomment next two lines if there is 64 GB system memory free
+; NumberOfBuffers          = 5450000
+; MaxDirtyBuffers          = 4000000
+;;
+;; Note the default settings will take very little memory
+;; but will not result in very good performance
+;;
+NumberOfBuffers          = 10000
+MaxDirtyBuffers          = 6000
+
+
+[HTTPServer]
+ServerPort			= 8804
+ServerRoot			= ../vsp
+MaxClientConnections		= 10
+DavRoot				= DAV
+EnabledDavVSP			= 0
+HTTPProxyEnabled		= 0
+TempASPXDir			= 0
+DefaultMailServer		= localhost:25
+MaxKeepAlives			= 10
+KeepAliveTimeout		= 10
+MaxCachedProxyConnections	= 10
+ProxyConnectionCacheTimeout	= 15
+HTTPThreadSize			= 280000
+HttpPrintWarningsInOutput	= 0
+Charset				= UTF-8
+;HTTPLogFile		        = logs/http.log
+MaintenancePage             	= atomic.html
+EnabledGzipContent          	= 1
+
+
+[AutoRepair]
+BadParentLinks			= 0
+
+[Client]
+SQL_PREFETCH_ROWS		= 100
+SQL_PREFETCH_BYTES		= 16000
+SQL_QUERY_TIMEOUT		= 0
+SQL_TXN_TIMEOUT			= 0
+;SQL_NO_CHAR_C_ESCAPE		= 1
+;SQL_UTF8_EXECS			= 0
+;SQL_NO_SYSTEM_TABLES		= 0
+;SQL_BINARY_TIMESTAMP		= 1
+;SQL_ENCRYPTION_ON_PASSWORD	= -1
+
+[VDB]
+ArrayOptimization		= 0
+NumArrayParameters		= 10
+VDBDisconnectTimeout		= 1000
+KeepConnectionOnFixedThread	= 0
+
+[Replication]
+ServerName			= db-CENTOS5-PORT
+ServerEnable			= 1
+QueueMax			= 50000
+
+
+;
+;  Striping setup
+;
+;  These parameters have only effect when Striping is set to 1 in the
+;  [Database] section, in which case the DatabaseFile parameter is ignored.
+;
+;  With striping, the database is spawned across multiple segments
+;  where each segment can have multiple stripes.
+;
+;  Format of the lines below:
+;    Segment<number> = <size>, <stripe file name> [, <stripe file name> .. ]
+;
+;  <number> must be ordered from 1 up.
+;
+;  The <size> is the total size of the segment which is equally divided
+;  across all stripes forming  the segment. Its specification can be in
+;  gigabytes (g), megabytes (m), kilobytes (k) or in database blocks
+;  (b, the default)
+;
+;  Note that the segment size must be a multiple of the database page size
+;  which is currently 8k. Also, the segment size must be divisible by the
+;  number of stripe files forming  the segment.
+;
+;  The example below creates a 200 meg database striped on two segments
+;  with two stripes of 50 meg and one of 100 meg.
+;
+;  You can always add more segments to the configuration, but once
+;  added, do not change the setup.
+;
+[Striping]
+Segment1			= 100M, db-seg1-1.db, db-seg1-2.db
+Segment2			= 100M, db-seg2-1.db
+;...
+
+;[TempStriping]
+;Segment1			= 100M, db-seg1-1.db, db-seg1-2.db
+;Segment2			= 100M, db-seg2-1.db
+;...
+
+;[Ucms]
+;UcmPath			= <path>
+;Ucm1				= <file>
+;Ucm2				= <file>
+;...
+
+
+[Zero Config]
+ServerName			= virtuoso (CENTOS5-PORT)
+;ServerDSN			= ZDSN
+;SSLServerName			= 
+;SSLServerDSN			= 
+
+
+[Mono]
+;MONO_TRACE			= Off
+;MONO_PATH			= <path_here>
+;MONO_ROOT			= <path_here>
+;MONO_CFG_DIR			= <path_here>
+;virtclr.dll			=
+
+
+[URIQA]
+DynamicLocal			= 0
+DefaultHost			= localhost:8890
+
+
+[SPARQL]
+;ExternalQuerySource		= 1
+;ExternalXsltSource 		= 1
+;DefaultGraph      		= http://localhost:8890/dataspace
+;ImmutableGraphs    		= http://localhost:8890/dataspace
+ResultSetMaxRows           	= 10000
+MaxConstructTriples		= 10000
+MaxQueryCostEstimationTime 	= 400	; in seconds
+MaxQueryExecutionTime      	= 60	; in seconds
+DefaultQuery               	= select distinct ?Concept where {[] a ?Concept} LIMIT 100
+DeferInferenceRulesInit    	= 0  ; controls inference rules loading
+MaxMemInUse			= 0  ; limits the amount of memory for construct dict (0=unlimited)
+;LabelInferenceName		= facets	; Only needed when using the Faceted Browser
+;PingService       		= http://rpc.pingthesemanticweb.com/
+
+
+[Plugins]
+LoadPath			= ../hosting
+Load1				= plain, wikiv
+Load2				= plain, mediawiki
+Load3				= plain, creolewiki
+Load4			= plain, im
+;Load5		= plain, wbxml2
+Load6		= plain, proj4
+Load7			= plain, geos
+Load8		= plain, shapefileio
+Load9		= plain, graphql
+;Load10			= plain, hslookup
+;Load11		= attach, libphp5.so
+;Load12		= Hosting, hosting_php.so
+;Load13		= Hosting,hosting_perl.so
+;Load14		= Hosting,hosting_python.so
+;Load15		= Hosting,hosting_ruby.so
+;Load16				= msdtc, msdtc_sample