Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into rpb-239-independentWo…
Browse files Browse the repository at this point in the history
…rks-import
  • Loading branch information
fsteeg committed Feb 25, 2025
2 parents cf27734 + 7dcaf83 commit ecc9464
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 22 deletions.
64 changes: 58 additions & 6 deletions app/controllers/nwbib/Application.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,12 @@
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.io.Streams;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.MapType;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;

Expand Down Expand Up @@ -506,6 +510,11 @@ private static Promise<Result> okPromise(final String q, final String person,
issued, medium, rpbspatial, rpbsubject, from, size, owner, t, sort,
details, location, word, corporation, raw, format);
return result.recover((Throwable throwable) -> {
Logger.error("Error on Lobid call with q={}, person={}, name={}, subject={}, id={}, publisher={},\n"
+ "issued={}, medium={}, rpbspatial={}, rpbsubject={}, from={}, size={}, owner={}, t={}, sort={},\n"
+ "details={}, location={}, word={}, corporation={}, raw={}, format={}", //
q, person, name, subject, id, publisher, issued, medium, rpbspatial, rpbsubject, from, size, owner,
t, sort, details, location, word, corporation, raw, format);
Logger.error("Could not call Lobid", throwable);
flashError();
return internalServerError(search.render("[]", q, person, name, subject,
Expand Down Expand Up @@ -976,16 +985,59 @@ private static Promise<Result> deleteFromIndex(String id) throws UnsupportedEnco

private static Promise<Result> transformAndIndex(String id, JsonNode jsonBody)
throws IOException, FileNotFoundException, RecognitionException, UnsupportedEncodingException {
JsonNode transformedJson = transform(jsonBody);
Promise<JsonNode> dataPromise = id.startsWith("f") && transformedJson.has("hbzId") ? // hbz-Fremddaten
addToLobidData(transformedJson) : Promise.pure(transformedJson);
return dataPromise.flatMap(result -> {
Cache.remove(String.format("/%s", id));
WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json");
return request.put(result).map(response -> status(response.getStatus(), response.getBody()));
});
}

private static JsonNode transform(JsonNode jsonBody)
throws IOException, FileNotFoundException, RecognitionException {
File input = new File("conf/output/test-output-strapi.json");
File output = new File("conf/output/test-output-0.json");
Files.write(Paths.get(input.getAbsolutePath()), jsonBody.toString().getBytes(Charsets.UTF_8));
ETL.main(new String[] {"conf/rpb-test-titel-to-lobid.flux"});
String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream().collect(Collectors.joining("\n"));
Cache.remove(String.format("/%s", id));
WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json");
return request.put(result).map(response -> status(response.getStatus(), response.getBody()));
ETL.main(new String[] { "conf/rpb-test-titel-to-lobid.flux" });
String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream()
.collect(Collectors.joining("\n"));
return Json.parse(result);
}

private static Promise<JsonNode> addToLobidData(JsonNode transformedJson) {
String lobidUrl = transformedJson.get("hbzId").textValue();
WSRequest lobidRequest = WS.url(lobidUrl).setHeader("Content-Type", "application/json");
Promise<JsonNode> lobidPromise = lobidRequest.get().map(WSResponse::asJson);
Promise<JsonNode> merged = lobidPromise.map(lobidJson -> mergeRecords(transformedJson, lobidJson));
return merged;
}


private static JsonNode mergeRecords(JsonNode transformedJson, JsonNode lobidJson)
throws JsonMappingException, JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
MapType mapType = TypeFactory.defaultInstance().constructMapType(Map.class, String.class, Object.class);
Map<String, Object> transformedMap = objectMapper.readValue(transformedJson.toString(), mapType);
Map<String, Object> lobidMap = objectMapper.readValue(lobidJson.toString(), mapType);
transformedMap.remove("type");
transformedMap.keySet().forEach(key -> {
Object transformedObject = transformedMap.get(key);
Object lobidObject = lobidMap.getOrDefault(key, new ArrayList<Object>());
Object values = transformedObject instanceof List ? mergeValues(transformedObject, lobidObject)
: transformedObject;
lobidMap.put(key, values);
});
return Json.toJson(lobidMap);
}

private static Object mergeValues(Object transformedObject, Object lobidObject) {
List<Object> mergedValues = lobidObject instanceof List ? new ArrayList<>((List<?>) lobidObject)
: Arrays.asList(lobidObject);
mergedValues.addAll((List<?>) transformedObject);
return mergedValues;
}

private static String elasticsearchUrl(String id) throws UnsupportedEncodingException {
return "http://weywot3:9200/resources-rpb-test/resource/"
+ URLEncoder.encode("https://lobid.org/resources/" + id, "UTF-8");
Expand Down
43 changes: 29 additions & 14 deletions app/views/TableRow.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
import controllers.nwbib.Classification;
import controllers.nwbib.Lobid;
import play.Logger;
import play.libs.F.Promise;
import play.libs.ws.WS;
import play.libs.ws.WSRequest;
import play.libs.ws.WSResponse;

/**
* Different ways of serializing a table row
Expand Down Expand Up @@ -107,27 +111,29 @@ public String process(JsonNode doc, String property, String param,
private String label(JsonNode doc, String value, List<String> properties) {
List<String> results = new ArrayList<>();
List<String> resultValues = labelsFor(doc, value, properties);
JsonNode labelNode = doc.get(properties.get(0)).iterator().next().get("label");
for (int i = 0; i < resultValues.size(); i++) {
String currentValue = resultValues.get(i);
String[] refAndLabel =
refAndLabel(properties.get(i), currentValue, Optional.empty());
String label = labelNode != null ? labelNode.textValue() : refAndLabel[1];
String result =
properties.get(i).equals("numbering") || value.equals("--")
? currentValue
: !value.startsWith("http") ? label : String.format(
"<a title=\"Titeldetails anzeigen\" href=\"%s\">%s</a>",
refAndLabel[0], label);
results.add(result.replace("Band", "").trim());
if (doc.get(properties.get(0)) != null) {
JsonNode labelNode = doc.get(properties.get(0)).iterator().next().get("label");
for (int i = 0; i < resultValues.size(); i++) {
String currentValue = resultValues.get(i);
String[] refAndLabel =
refAndLabel(properties.get(i), currentValue, Optional.empty());
String label = labelNode != null ? labelNode.textValue() : refAndLabel[1];
String result =
properties.get(i).equals("numbering") || value.equals("--")
? currentValue
: !value.startsWith("http") ? label : String.format(
"<a title=\"Titeldetails anzeigen\" href=\"%s\">%s</a>",
refAndLabel[0], label);
results.add(result.replace("Band", "").trim());
}
}
return results.stream().collect(Collectors.joining(", Band "));
}

private List<String> labelsFor(JsonNode doc, String value,
List<String> keys) {
List<String> result = new ArrayList<>();
if (doc != null) {
if (doc != null && doc.get(keys.get(0)) != null) {
JsonNode node = doc.get(keys.get(0)).iterator().next();
JsonNode id = node.get("id");
JsonNode label = node.get("label");
Expand Down Expand Up @@ -214,6 +220,9 @@ private static String lifeDates(JsonNode node) {

String[] refAndLabel(String property, String value,
Optional<List<String>> labels) {
if (value.contains("lobid.org/resources/")) {
value = rpbUrlIfInRpb(value);
}
if ((property.equals("containedIn") || property.equals("hasPart")
|| property.equals("isPartOf") || property.equals("hasSuperordinate")
|| property.equals("bibliographicCitation")) && value.contains("lobid.org")) {
Expand All @@ -227,6 +236,12 @@ String[] refAndLabel(String property, String value,
return new String[] { value, label };
}

String rpbUrlIfInRpb(String value) {
WSRequest lobidRequest = WS.url(value).setHeader("Content-Type", "application/json");
JsonNode lobidJson = lobidRequest.get().map(WSResponse::asJson).get(Lobid.API_TIMEOUT);
return lobidJson.has("rpbId") ? "https://rpb.lobid.org/" + lobidJson.get("rpbId").textValue() : value;
}

public abstract String process(JsonNode doc, String property, String param,
String label, List<String> values, Optional<List<String>> labels);
}
2 changes: 1 addition & 1 deletion conf/nwbib.conf
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ medium.labels={
"http://rdaregistry.info/termList/RDAMediaType/1002" : ["Mikroform","octicon octicon-versions", 4],
"http://purl.org/ontology/bibo/AudioDocument" : ["Audio","glyphicon glyphicon-headphones", 5],
"http://purl.org/ontology/bibo/AudioVisualDocument" : ["Audio-Visuell","glyphicon glyphicon-film", 6],
"http://rdaregistry.info/termList/RDAMediaType/1008" : ["Kombination","glyphicon glyphicon-asterisk", 7],
"http://rdaregistry.info/termList/RDAMediaType/1008" : ["Video","glyphicon glyphicon-asterisk", 7],
"http://purl.org/library/BrailleBook" : ["Braille","glyphicon glyphicon-eye-close", 8],
"http://purl.org/ontology/mo/Vinyl" : ["Schallplatte","glyphicon glyphicon-record", 9],
"http://id.loc.gov/vocabulary/carriers/ss" : ["Tonband", "octicon octicon-link", 10],
Expand Down
2 changes: 1 addition & 1 deletion monit_restart.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ case $ACTION in
kill $(cat target/universal/stage/RUNNING_PID)
rm target/universal/stage/RUNNING_PID
fi
JAVA_OPTS="$JAVA_OPTS -XX:+HeapDumpOnOutOfMemoryError" $HOME/activator-dist-1.3.5/activator "start $PORT"
JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" sbt "start $PORT"
;;
stop)
kill $(cat target/universal/stage/RUNNING_PID)
Expand Down

0 comments on commit ecc9464

Please sign in to comment.