From 90f599a1089e1d9fea68ca1d25cd158bbabe4705 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 7 Jan 2025 10:40:11 +0100 Subject: [PATCH 1/9] Handle external records with `hbzId` in PUT route (RPB-184) Merge transformed strapi data with linked lobid record for indexing --- app/controllers/nwbib/Application.java | 58 +++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/app/controllers/nwbib/Application.java b/app/controllers/nwbib/Application.java index b3e5a74..971110b 100644 --- a/app/controllers/nwbib/Application.java +++ b/app/controllers/nwbib/Application.java @@ -37,8 +37,12 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.Streams; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.MapType; +import com.fasterxml.jackson.databind.type.TypeFactory; import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; @@ -976,16 +980,58 @@ private static Promise deleteFromIndex(String id) throws UnsupportedEnco private static Promise transformAndIndex(String id, JsonNode jsonBody) throws IOException, FileNotFoundException, RecognitionException, UnsupportedEncodingException { + JsonNode transformedJson = transform(jsonBody); + Promise dataPromise = id.startsWith("f") && transformedJson.has("hbzId") ? // hbz-Fremddaten + addToLobidData(transformedJson) : Promise.pure(transformedJson); + return dataPromise.flatMap(result -> { + Cache.remove(String.format("/%s", id)); + WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json"); + return request.put(result).map(response -> status(response.getStatus(), response.getBody())); + }); + } + + private static JsonNode transform(JsonNode jsonBody) + throws IOException, FileNotFoundException, RecognitionException { File input = new File("conf/output/test-output-strapi.json"); File output = new File("conf/output/test-output-0.json"); Files.write(Paths.get(input.getAbsolutePath()), jsonBody.toString().getBytes(Charsets.UTF_8)); - ETL.main(new String[] {"conf/rpb-test-titel-to-lobid.flux"}); - String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream().collect(Collectors.joining("\n")); - Cache.remove(String.format("/%s", id)); - WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json"); - return request.put(result).map(response -> status(response.getStatus(), response.getBody())); + ETL.main(new String[] { "conf/rpb-test-titel-to-lobid.flux" }); + String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream() + .collect(Collectors.joining("\n")); + return Json.parse(result); + } + + private static Promise addToLobidData(JsonNode transformedJson) { + String lobidUrl = transformedJson.get("hbzId").textValue(); + WSRequest lobidRequest = WS.url(lobidUrl).setHeader("Content-Type", "application/json"); + Promise lobidPromise = lobidRequest.get().map(WSResponse::asJson); + Promise merged = lobidPromise.map(lobidJson -> mergeRecords(transformedJson, lobidJson)); + return merged; } - + + private static JsonNode mergeRecords(JsonNode transformedJson, JsonNode lobidJson) + throws JsonMappingException, JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + MapType mapType = TypeFactory.defaultInstance().constructMapType(Map.class, String.class, Object.class); + Map transformedMap = objectMapper.readValue(transformedJson.toString(), mapType); + Map lobidMap = objectMapper.readValue(lobidJson.toString(), mapType); + transformedMap.keySet().forEach(key -> { + Object transformedObject = transformedMap.get(key); + Object lobidObject = lobidMap.getOrDefault(key, new ArrayList()); + Object values = transformedObject instanceof List ? mergeValues(transformedObject, lobidObject) + : transformedObject; + lobidMap.put(key, values); + }); + return Json.toJson(lobidMap); + } + + private static Object mergeValues(Object transformedObject, Object lobidObject) { + List mergedValues = lobidObject instanceof List ? new ArrayList<>((List) lobidObject) + : Arrays.asList(lobidObject); + mergedValues.addAll((List) transformedObject); + return mergedValues; + } + private static String elasticsearchUrl(String id) throws UnsupportedEncodingException { return "http://weywot3:9200/resources-rpb-test/resource/" + URLEncoder.encode("https://lobid.org/resources/" + id, "UTF-8"); From 32b5761e02cc80522ff5de4a8d624862d8d47c18 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 14 Jan 2025 16:18:03 +0100 Subject: [PATCH 2/9] Update label for video, use only lobid types when merging (RPB-184) See also RPB-250 and RPB-251 --- app/controllers/nwbib/Application.java | 1 + conf/nwbib.conf | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/nwbib/Application.java b/app/controllers/nwbib/Application.java index 971110b..1f05c6c 100644 --- a/app/controllers/nwbib/Application.java +++ b/app/controllers/nwbib/Application.java @@ -1015,6 +1015,7 @@ private static JsonNode mergeRecords(JsonNode transformedJson, JsonNode lobidJso MapType mapType = TypeFactory.defaultInstance().constructMapType(Map.class, String.class, Object.class); Map transformedMap = objectMapper.readValue(transformedJson.toString(), mapType); Map lobidMap = objectMapper.readValue(lobidJson.toString(), mapType); + transformedMap.remove("type"); transformedMap.keySet().forEach(key -> { Object transformedObject = transformedMap.get(key); Object lobidObject = lobidMap.getOrDefault(key, new ArrayList()); diff --git a/conf/nwbib.conf b/conf/nwbib.conf index 23343b8..c1696be 100644 --- a/conf/nwbib.conf +++ b/conf/nwbib.conf @@ -64,7 +64,7 @@ medium.labels={ "http://rdaregistry.info/termList/RDAMediaType/1002" : ["Mikroform","octicon octicon-versions", 4], "http://purl.org/ontology/bibo/AudioDocument" : ["Audio","glyphicon glyphicon-headphones", 5], "http://purl.org/ontology/bibo/AudioVisualDocument" : ["Audio-Visuell","glyphicon glyphicon-film", 6], - "http://rdaregistry.info/termList/RDAMediaType/1008" : ["Kombination","glyphicon glyphicon-asterisk", 7], + "http://rdaregistry.info/termList/RDAMediaType/1008" : ["Video","glyphicon glyphicon-asterisk", 7], "http://purl.org/library/BrailleBook" : ["Braille","glyphicon glyphicon-eye-close", 8], "http://purl.org/ontology/mo/Vinyl" : ["Schallplatte","glyphicon glyphicon-record", 9], "http://id.loc.gov/vocabulary/carriers/ss" : ["Tonband", "octicon octicon-link", 10], From c0dd153d87e48710b87c7cb892a5c1ffdd919903 Mon Sep 17 00:00:00 2001 From: Phu Tu Date: Fri, 24 Jan 2025 12:06:35 +0100 Subject: [PATCH 3/9] Change location of older application logs. Resolves #123 --- conf/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/logback.xml b/conf/logback.xml index 5d2754a..14bf764 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -10,7 +10,7 @@ ${application.home}/logs/application.log - application-log-%d{yyyy-ww}.gz + ${application.home}/logs/application-log-%d{yyyy-ww}.gz 8 From 4e6ab28c251532ddae4403e0c479ceb7ebb22fa7 Mon Sep 17 00:00:00 2001 From: Phu Tu Date: Fri, 24 Jan 2025 12:10:21 +0100 Subject: [PATCH 4/9] Revert "Change location of older application logs. Resolves #123" This reverts commit c0dd153d87e48710b87c7cb892a5c1ffdd919903. --- conf/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/logback.xml b/conf/logback.xml index 14bf764..5d2754a 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -10,7 +10,7 @@ ${application.home}/logs/application.log - ${application.home}/logs/application-log-%d{yyyy-ww}.gz + application-log-%d{yyyy-ww}.gz 8 From f193807ee75bee8541a9dcf0264a73ebede895b8 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Fri, 24 Jan 2025 14:38:45 +0100 Subject: [PATCH 5/9] Replace specific activator version with `sbt` command (RPB-245) --- monit_restart.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monit_restart.sh b/monit_restart.sh index d3ed6a4..c323ae0 100755 --- a/monit_restart.sh +++ b/monit_restart.sh @@ -34,7 +34,7 @@ case $ACTION in kill $(cat target/universal/stage/RUNNING_PID) rm target/universal/stage/RUNNING_PID fi - JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError" $HOME/activator-dist-1.3.5/activator "start $PORT" + JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError" sbt "start $PORT" ;; stop) kill $(cat target/universal/stage/RUNNING_PID) From 149cb38ab48a94a54cb53e3282779793865a7bf0 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Fri, 31 Jan 2025 15:20:20 +0100 Subject: [PATCH 6/9] Replace lobid URL in isPartOf link with RPB URL (RPB-184) If the linked resource has an `rpbId` --- app/views/TableRow.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/views/TableRow.java b/app/views/TableRow.java index de0672b..92e3a92 100644 --- a/app/views/TableRow.java +++ b/app/views/TableRow.java @@ -18,6 +18,10 @@ import controllers.nwbib.Classification; import controllers.nwbib.Lobid; import play.Logger; +import play.libs.F.Promise; +import play.libs.ws.WS; +import play.libs.ws.WSRequest; +import play.libs.ws.WSResponse; /** * Different ways of serializing a table row @@ -214,6 +218,9 @@ private static String lifeDates(JsonNode node) { String[] refAndLabel(String property, String value, Optional> labels) { + if (value.contains("lobid.org/resources/")) { + value = rpbUrlIfInRpb(value); + } if ((property.equals("containedIn") || property.equals("hasPart") || property.equals("isPartOf") || property.equals("hasSuperordinate") || property.equals("bibliographicCitation")) && value.contains("lobid.org")) { @@ -227,6 +234,12 @@ String[] refAndLabel(String property, String value, return new String[] { value, label }; } + String rpbUrlIfInRpb(String value) { + WSRequest lobidRequest = WS.url(value).setHeader("Content-Type", "application/json"); + JsonNode lobidJson = lobidRequest.get().map(WSResponse::asJson).get(Lobid.API_TIMEOUT); + return lobidJson.has("rpbId") ? "https://rpb.lobid.org/" + lobidJson.get("rpbId").textValue() : value; + } + public abstract String process(JsonNode doc, String property, String param, String label, List values, Optional> labels); } \ No newline at end of file From 6f22eb5ca207fd7d9e17918951f48728486dc8a3 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Mon, 3 Feb 2025 14:01:52 +0100 Subject: [PATCH 7/9] Add logging statement to find records causing errors (RPB-255) --- app/controllers/nwbib/Application.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/controllers/nwbib/Application.java b/app/controllers/nwbib/Application.java index b3e5a74..2f31d16 100644 --- a/app/controllers/nwbib/Application.java +++ b/app/controllers/nwbib/Application.java @@ -506,6 +506,11 @@ private static Promise okPromise(final String q, final String person, issued, medium, rpbspatial, rpbsubject, from, size, owner, t, sort, details, location, word, corporation, raw, format); return result.recover((Throwable throwable) -> { + Logger.error("Error on Lobid call with q={}, person={}, name={}, subject={}, id={}, publisher={},\n" + + "issued={}, medium={}, rpbspatial={}, rpbsubject={}, from={}, size={}, owner={}, t={}, sort={},\n" + + "details={}, location={}, word={}, corporation={}, raw={}, format={}", // + q, person, name, subject, id, publisher, issued, medium, rpbspatial, rpbsubject, from, size, owner, + t, sort, details, location, word, corporation, raw, format); Logger.error("Could not call Lobid", throwable); flashError(); return internalServerError(search.render("[]", q, person, name, subject, From 178a2bbfeaa0283a06968d0c13e7b727f195a6de Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Fri, 14 Feb 2025 11:22:06 +0100 Subject: [PATCH 8/9] Handle missing data in details view (RPB-255) --- app/views/TableRow.java | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/app/views/TableRow.java b/app/views/TableRow.java index ee24988..714e508 100644 --- a/app/views/TableRow.java +++ b/app/views/TableRow.java @@ -111,19 +111,21 @@ public String process(JsonNode doc, String property, String param, private String label(JsonNode doc, String value, List properties) { List results = new ArrayList<>(); List resultValues = labelsFor(doc, value, properties); - JsonNode labelNode = doc.get(properties.get(0)).iterator().next().get("label"); - for (int i = 0; i < resultValues.size(); i++) { - String currentValue = resultValues.get(i); - String[] refAndLabel = - refAndLabel(properties.get(i), currentValue, Optional.empty()); - String label = labelNode != null ? labelNode.textValue() : refAndLabel[1]; - String result = - properties.get(i).equals("numbering") || value.equals("--") - ? currentValue - : String.format( - "%s", - refAndLabel[0], label); - results.add(result.replace("Band", "").trim()); + if (doc.get(properties.get(0)) != null) { + JsonNode labelNode = doc.get(properties.get(0)).iterator().next().get("label"); + for (int i = 0; i < resultValues.size(); i++) { + String currentValue = resultValues.get(i); + String[] refAndLabel = + refAndLabel(properties.get(i), currentValue, Optional.empty()); + String label = labelNode != null ? labelNode.textValue() : refAndLabel[1]; + String result = + properties.get(i).equals("numbering") || value.equals("--") + ? currentValue + : String.format( + "%s", + refAndLabel[0], label); + results.add(result.replace("Band", "").trim()); + } } return results.stream().collect(Collectors.joining(", Band ")); } @@ -131,7 +133,7 @@ private String label(JsonNode doc, String value, List properties) { private List labelsFor(JsonNode doc, String value, List keys) { List result = new ArrayList<>(); - if (doc != null) { + if (doc != null && doc.get(keys.get(0)) != null) { JsonNode node = doc.get(keys.get(0)).iterator().next(); JsonNode id = node.get("id"); JsonNode label = node.get("label"); From 7dcaf837f577fb1e771b1fafbd6110d403826633 Mon Sep 17 00:00:00 2001 From: Pascal Christoph Date: Thu, 20 Feb 2025 11:58:36 +0100 Subject: [PATCH 9/9] Remove HeapDumpOnOutOfMemoryError java option As there is a memory leak in metafacture-core#666 several instances dump 32 GB sized heap profiles. Our server have not enough space to handle these. It's btter to enable this option when needed, not as default. --- monit_restart.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monit_restart.sh b/monit_restart.sh index c323ae0..58899ca 100755 --- a/monit_restart.sh +++ b/monit_restart.sh @@ -34,7 +34,7 @@ case $ACTION in kill $(cat target/universal/stage/RUNNING_PID) rm target/universal/stage/RUNNING_PID fi - JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError" sbt "start $PORT" + JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" sbt "start $PORT" ;; stop) kill $(cat target/universal/stage/RUNNING_PID)