Merge pull request #561 from hobbit-project/feature-limit-resources

Feature limit resources
hobbit-project · Dec 18, 2023 · f087c48 · f087c48
2 parents 86893cb + ff5d0c1
commit f087c48
Show file tree

Hide file tree

Showing 56 changed files with 893 additions and 637 deletions.
diff --git a/analysis-component/pom.xml b/analysis-component/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.hobbit</groupId>
         <artifactId>parent</artifactId>
-        <version>${hobbitplatform.version}</version>
+        <version>2.0.17</version>
         <relativePath>../parent-pom</relativePath>
     </parent>
     <artifactId>analysis-component</artifactId>

diff --git a/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java b/analysis-component/src/main/java/org/hobbit/analysis/AnalysisComponent.java
@@ -16,22 +16,42 @@
  */
 package org.hobbit.analysis;
 
-import java.util.stream.Stream;
-import java.util.stream.Collectors;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.aksw.palmetto.evaluate.correlation.PearsonsSampleCorrelationCoefficient;
+import org.apache.commons.io.IOUtils;
 import org.apache.jena.datatypes.RDFDatatype;
 import org.apache.jena.datatypes.xsd.XSDDatatype;
-import org.apache.jena.rdf.model.*;
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.jena.rdf.model.Property;
+import org.apache.jena.rdf.model.RDFNode;
+import org.apache.jena.rdf.model.ResIterator;
+import org.apache.jena.rdf.model.Resource;
+import org.apache.jena.rdf.model.ResourceFactory;
+import org.apache.jena.rdf.model.Statement;
+import org.apache.jena.rdf.model.StmtIterator;
 import org.apache.jena.vocabulary.DCTerms;
 import org.apache.jena.vocabulary.RDF;
 import org.apache.jena.vocabulary.XSD;
-import org.apache.commons.io.IOUtils;
 import org.hobbit.core.Constants;
 import org.hobbit.core.components.AbstractComponent;
 import org.hobbit.core.data.RabbitQueue;
+import org.hobbit.core.rabbit.QueueingConsumer;
 import org.hobbit.core.rabbit.RabbitMQUtils;
 import org.hobbit.storage.client.StorageServiceClient;
 import org.hobbit.storage.queries.SparqlQueries;
@@ -42,20 +62,31 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.rabbitmq.client.QueueingConsumer;
+import com.rabbitmq.client.Delivery;
 
-import weka.attributeSelection.*;
 import weka.attributeSelection.AttributeSelection;
+import weka.attributeSelection.CfsSubsetEval;
+import weka.attributeSelection.GreedyStepwise;
 import weka.classifiers.functions.LinearRegression;
 import weka.clusterers.SimpleKMeans;
-import weka.core.*;
+import weka.core.Attribute;
+import weka.core.DenseInstance;
+import weka.core.Instance;
+import weka.core.Instances;
+import weka.core.SparseInstance;
 
 
 /**
  * This class implements the functionality for the Analysis Component
  * TODO:: !!REFACTOR INTO A MORE GENERIC DESIGN!!
  */
 public class AnalysisComponent extends AbstractComponent {
+    /**
+     * The time the main receiver thread will sleep if it didn't receive any message
+     * (in seconds).
+     */
+    private static final int WAITING_TIME_BEFORE_CHECKING_STATUS = 60;
+
     private static final Logger LOGGER = LoggerFactory.getLogger(AnalysisComponent.class);
     private static final String GRAPH_URI = Constants.PUBLIC_RESULT_GRAPH_URI;
     protected RabbitQueue controller2AnalysisQueue;
@@ -82,16 +113,24 @@ public void init() throws Exception {
     @Override
     public void run() throws Exception {
         LOGGER.info("Awaiting requests");
-        QueueingConsumer.Delivery delivery;
+        Delivery delivery;
         while (true) {
-            delivery = consumer.nextDelivery();
+            delivery = null;
+            // Let's wait for a delivery for 60 seconds
+            try {
+                delivery = consumer.getDeliveryQueue().poll(WAITING_TIME_BEFORE_CHECKING_STATUS, TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+                // interrupted; just continue
+            }
             if (delivery != null) {
                 LOGGER.info("Received a request. Processing...");
                 String expUri = RabbitMQUtils.readString(delivery.getBody());
                 handleRequest(expUri);
+            } else {
+                // This would be the place at which we could react to signals, e.g., terminate
+                // the service if needed.
             }
         }
-
     }
 
     protected void handleRequest(String expUri) {
@@ -235,18 +274,17 @@ public void close() throws IOException {
      *            The model of the experiment
      * @return Returns the analyzed model
      */
-
-    /**
+    /*
      private AnalysisModel analyseExperiment(Model experimentModel, String expUri){
      AnalysisModel analysisModel = new AnalysisModel(experimentModel, expUri);
      analysisModel.analyse();
      return analysisModel;
      }*/
 
-    private void notifyQueue(){
+    /*private void notifyQueue(){
         //TODO:: return the status of component
 
-    }
+    }*/
 
     /**
      * This class implements the functionality of the Analysis Model which

diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
@@ -9,16 +9,20 @@ services:
       HOBBIT_RABBIT_IMAGE: "rabbitmq:management"
       HOBBIT_RABBIT_HOST: "rabbit"
       HOBBIT_REDIS_HOST: "redis"
-      DEPLOY_ENV: "testing"
+      DEPLOY_ENV: "develop"
       GITLAB_USER: "${GITLAB_USER}"
       GITLAB_EMAIL: "${GITLAB_EMAIL}"
       GITLAB_TOKEN: "${GITLAB_TOKEN}"
-      LOGGING_GELF_ADDRESS: "udp://localhost:12201"
+      #LOGGING_GELF_ADDRESS: "udp://localhost:12201"
       SWARM_NODE_NUMBER: "1"
       PROMETHEUS_HOST: prometheus
       PROMETHEUS_PORT: 9090
+      USE_GITLAB: "false"
+      LOCAL_METADATA_DIRECTORY: "/metadata"
+      DOCKER_AUTOPULL: "0"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      - ./metadata:/metadata
 
   # HOBBIT GUI
   gui:
@@ -37,8 +41,7 @@ services:
       - ELASTICSEARCH_HOST=elasticsearch
       - ELASTICSEARCH_HTTP_PORT=9200
       - USE_UI_AUTH=false
-    volumes:
-      - ./config/jetty/web-without-ui-auth.xml:/var/lib/jetty/webapps/ROOT/WEB-INF/web.xml
+    #volumes:
       #- /data/docker/messages/global.html:/var/lib/jetty/webapps/messages/global.html
       #- /data/docker/messages/benchmark.html:/var/lib/jetty/webapps/messages/benchmark.html
       #- /data/docker/messages/status.html:/var/lib/jetty/webapps/messages/status.html
@@ -92,7 +95,8 @@ services:
     stop_signal: SIGINT
     stop_grace_period: 2m
     volumes:
-      - ./config/db:/opt/virtuoso-opensource/database
+      - ./config/db:/opt/virtuoso-opensource/var/lib/virtuoso/db
+      #- ./config/db:/opt/virtuoso-opensource/database
     networks:
       - hobbit-core
     ports:
@@ -106,8 +110,8 @@ services:
     environment:
       - SPARQL_ENDPOINT_URL=http://vos:8890/sparql
       - HOBBIT_RABBIT_HOST=rabbit
-      - SPARQL_ENDPOINT_USERNAME=HobbitPlatform
-      - SPARQL_ENDPOINT_PASSWORD=Password
+      - SPARQL_ENDPOINT_USERNAME=dba
+      - SPARQL_ENDPOINT_PASSWORD=dba
 
   node-exporter:
     image: prom/node-exporter

diff --git a/hobbit-gui/gui-serverbackend/Dockerfile b/hobbit-gui/gui-serverbackend/Dockerfile
@@ -3,13 +3,15 @@
 # https://www.npmjs.com/package/node-sass#node-version-support-policy
 # node-sass 4.10.0 (../gui-client/package-lock.json) ⇒ node <= 11
 
+# Build web pages
 FROM node:11 AS build-client
 WORKDIR /usr/src/hobbit-platform
 COPY hobbit-gui/gui-client/package*.json ./hobbit-gui/gui-client/
 RUN npm --prefix hobbit-gui/gui-client ci --omit=dev
 COPY hobbit-gui/gui-client ./hobbit-gui/gui-client
 RUN npm --prefix hobbit-gui/gui-client run build-prod
 
+# Build Java backend
 FROM maven:3-eclipse-temurin-11 AS build
 WORKDIR /usr/src/hobbit-platform
 COPY parent-pom/pom.xml ./parent-pom/
@@ -20,14 +22,19 @@ COPY hobbit-gui/gui-serverbackend/src ./hobbit-gui/gui-serverbackend/src
 COPY --from=build-client /usr/src/hobbit-platform/hobbit-gui/gui-client/dist ./hobbit-gui/gui-client/dist
 RUN mvn --file ./hobbit-gui/gui-serverbackend/ -Dmaven.test.skip=true package
 
+# Set up web server
 FROM jetty:9.3-jre8
-
+# Add script to register keycloak
+COPY hobbit-gui/gui-serverbackend/configure-auth.sh $JETTY_BASE/configure-auth.sh
+# Download the keycloak adapter and add it
 RUN cd $JETTY_BASE && \
  curl -L -O http://hobbitdata.informatik.uni-leipzig.de/hobbit/keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip && \
  unzip keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip  && \
  rm -f keycloak-jetty93-adapter-for-hobbit-dist-2.4.0.Final.zip  && \
  java -jar $JETTY_HOME/start.jar --add-to-startd=keycloak
-
+# Copy message definitions
 COPY hobbit-gui/gui-serverbackend/messages /var/lib/jetty/webapps/messages
+# Copy Java backend
+COPY --chown=jetty --chmod=777 --from=build /usr/src/hobbit-platform/hobbit-gui/gui-serverbackend/target/gui-serverbackend $JETTY_BASE/webapps/ROOT
 
-COPY --from=build /usr/src/hobbit-platform/hobbit-gui/gui-serverbackend/target/gui-serverbackend $JETTY_BASE/webapps/ROOT
+ENTRYPOINT ./configure-auth.sh && /docker-entrypoint.sh
diff --git a/hobbit-gui/gui-serverbackend/configure-auth.sh b/hobbit-gui/gui-serverbackend/configure-auth.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# Runs the GUI serverbackend of the HOBBIT Platform.
+USE_KEYCLOAK="${USE_UI_AUTH:-true}"
+
+if [ "${USE_KEYCLOAK}" == "false" ]; then
+    echo "Replacing web.xml with web-without-ui-auth.xml"
+    cp /var/lib/jetty/webapps/ROOT/WEB-INF/web-without-ui-auth.xml /var/lib/jetty/webapps/ROOT/WEB-INF/web.xml
+fi
+
diff --git a/hobbit-gui/gui-serverbackend/pom.xml b/hobbit-gui/gui-serverbackend/pom.xml
@@ -14,7 +14,7 @@
     <parent>
         <groupId>org.hobbit</groupId>
         <artifactId>parent</artifactId>
-        <version>${hobbitplatform.version}</version>
+        <version>2.0.17</version>
         <relativePath>../../parent-pom</relativePath>
     </parent>
     <artifactId>gui-serverbackend</artifactId>