Skip to content

Commit

Permalink
Merge branch 'main' of https://github.com/hbz/rpb into biblioVino
Browse files Browse the repository at this point in the history
  • Loading branch information
@fsteeg
fsteeg committed Feb 10, 2025
2 parents 82163f4 + fb65fbf commit 9a5b411
Show file tree
Hide file tree
Showing 21 changed files with 192,588 additions and 192,431 deletions.
64 changes: 58 additions & 6 deletions app/controllers/nwbib/Application.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,12 @@
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.io.Streams;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.MapType;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;

Expand Down Expand Up @@ -506,6 +510,11 @@ private static Promise<Result> okPromise(final String q, final String person,
issued, medium, rpbspatial, rpbsubject, from, size, owner, t, sort,
details, location, word, corporation, raw, format);
return result.recover((Throwable throwable) -> {
Logger.error("Error on Lobid call with q={}, person={}, name={}, subject={}, id={}, publisher={},\n"
+ "issued={}, medium={}, rpbspatial={}, rpbsubject={}, from={}, size={}, owner={}, t={}, sort={},\n"
+ "details={}, location={}, word={}, corporation={}, raw={}, format={}", //
q, person, name, subject, id, publisher, issued, medium, rpbspatial, rpbsubject, from, size, owner,
t, sort, details, location, word, corporation, raw, format);
Logger.error("Could not call Lobid", throwable);
flashError();
return internalServerError(search.render("[]", q, person, name, subject,
Expand Down Expand Up @@ -976,16 +985,59 @@ private static Promise<Result> deleteFromIndex(String id) throws UnsupportedEnco

private static Promise<Result> transformAndIndex(String id, JsonNode jsonBody)
throws IOException, FileNotFoundException, RecognitionException, UnsupportedEncodingException {
JsonNode transformedJson = transform(jsonBody);
Promise<JsonNode> dataPromise = id.startsWith("f") && transformedJson.has("hbzId") ? // hbz-Fremddaten
addToLobidData(transformedJson) : Promise.pure(transformedJson);
return dataPromise.flatMap(result -> {
Cache.remove(String.format("/%s", id));
WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json");
return request.put(result).map(response -> status(response.getStatus(), response.getBody()));
});
}

private static JsonNode transform(JsonNode jsonBody)
throws IOException, FileNotFoundException, RecognitionException {
File input = new File("conf/output/test-output-strapi.json");
File output = new File("conf/output/test-output-0.json");
Files.write(Paths.get(input.getAbsolutePath()), jsonBody.toString().getBytes(Charsets.UTF_8));
ETL.main(new String[] {"conf/rpb-test-titel-to-lobid.flux"});
String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream().collect(Collectors.joining("\n"));
Cache.remove(String.format("/%s", id));
WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json");
return request.put(result).map(response -> status(response.getStatus(), response.getBody()));
ETL.main(new String[] { "conf/rpb-test-titel-to-lobid.flux" });
String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream()
.collect(Collectors.joining("\n"));
return Json.parse(result);
}

private static Promise<JsonNode> addToLobidData(JsonNode transformedJson) {
String lobidUrl = transformedJson.get("hbzId").textValue();
WSRequest lobidRequest = WS.url(lobidUrl).setHeader("Content-Type", "application/json");
Promise<JsonNode> lobidPromise = lobidRequest.get().map(WSResponse::asJson);
Promise<JsonNode> merged = lobidPromise.map(lobidJson -> mergeRecords(transformedJson, lobidJson));
return merged;
}


private static JsonNode mergeRecords(JsonNode transformedJson, JsonNode lobidJson)
throws JsonMappingException, JsonProcessingException {
ObjectMapper objectMapper = new ObjectMapper();
MapType mapType = TypeFactory.defaultInstance().constructMapType(Map.class, String.class, Object.class);
Map<String, Object> transformedMap = objectMapper.readValue(transformedJson.toString(), mapType);
Map<String, Object> lobidMap = objectMapper.readValue(lobidJson.toString(), mapType);
transformedMap.remove("type");
transformedMap.keySet().forEach(key -> {
Object transformedObject = transformedMap.get(key);
Object lobidObject = lobidMap.getOrDefault(key, new ArrayList<Object>());
Object values = transformedObject instanceof List ? mergeValues(transformedObject, lobidObject)
: transformedObject;
lobidMap.put(key, values);
});
return Json.toJson(lobidMap);
}

private static Object mergeValues(Object transformedObject, Object lobidObject) {
List<Object> mergedValues = lobidObject instanceof List ? new ArrayList<>((List<?>) lobidObject)
: Arrays.asList(lobidObject);
mergedValues.addAll((List<?>) transformedObject);
return mergedValues;
}

private static String elasticsearchUrl(String id) throws UnsupportedEncodingException {
return "http://weywot3:9200/resources-rpb-test/resource/"
+ URLEncoder.encode("https://lobid.org/resources/" + id, "UTF-8");
Expand Down
17 changes: 15 additions & 2 deletions app/views/TableRow.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
import controllers.nwbib.Classification;
import controllers.nwbib.Lobid;
import play.Logger;
import play.libs.F.Promise;
import play.libs.ws.WS;
import play.libs.ws.WSRequest;
import play.libs.ws.WSResponse;

/**
* Different ways of serializing a table row
Expand Down Expand Up @@ -214,11 +218,14 @@ private static String lifeDates(JsonNode node) {

String[] refAndLabel(String property, String value,
Optional<List<String>> labels) {
if (value.contains("lobid.org/resources/")) {
value = rpbUrlIfInRpb(value);
}
if ((property.equals("containedIn") || property.equals("hasPart")
|| property.equals("isPartOf") || property.equals("hasSuperordinate")
|| property.equals("bibliographicCitation")) && value.contains("lobid.org")) {
return new String[] { value.matches(".*?[as]\\d+.*|.*?\\d{3}[a-z]\\d+.*") // rpbId
? value.replace("https://lobid.org/resources/", "/")
return new String[] { value.matches(".*?[asf]\\d+.*|.*?\\d{3}[a-z]\\d+.*") // rpbId
? value.replaceAll("http.+/", "/") // full URL -> relative link
: value, Lobid.resourceLabel(value) };
}
String label =
Expand All @@ -227,6 +234,12 @@ String[] refAndLabel(String property, String value,
return new String[] { value, label };
}

String rpbUrlIfInRpb(String value) {
WSRequest lobidRequest = WS.url(value).setHeader("Content-Type", "application/json");
JsonNode lobidJson = lobidRequest.get().map(WSResponse::asJson).get(Lobid.API_TIMEOUT);
return lobidJson.has("rpbId") ? "https://rpb.lobid.org/" + lobidJson.get("rpbId").textValue() : value;
}

public abstract String process(JsonNode doc, String property, String param,
String label, List<String> values, Optional<List<String>> labels);
}
28 changes: 17 additions & 11 deletions app/views/tags/result_doc.scala.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,10 @@
}
}

@subordinate(field:String, id:String, label:String, values:(String,String)) = {
@defining(Lobid.getTotalHits(field, id, CONFIG.getString("nwbib.filter")).get(Lobid.API_TIMEOUT)){ hits => @if(hits > 0) {
@subordinate(field:String, ids:Seq[String], label:String, values:(String,String)) = {
@defining(ids.map((id) => Lobid.getTotalHits(field, id, CONFIG.getString("nwbib.filter")).get(Lobid.API_TIMEOUT)).reduce((a, b) => a + b)){ hits => @if(hits > 0) {
@defining(if(hits==1){values._1} else {values._2}){ value =>
<tr><td>@label</td><td><a title="@value" href="@nwbib.routes.Application.search(raw=field+":\""+Lobid.escapeUri(id)+"\"")">@hits @value</a></td></tr>
<tr><td>@label</td><td><a title="@value" href="@nwbib.routes.Application.search(raw=ids.map((id)=>field+":\""+Lobid.escapeUri(id)+"\"").mkString(" OR "))">@hits @value</a></td></tr>
}}}
}

Expand Down Expand Up @@ -148,7 +148,10 @@
component <- components;
label <- (component \ "label").asOpt[String];
id = (component \ "id").asOpt[String].getOrElse("")) {
<span style="white-space: nowrap;"><a title="Nach weiteren Titeln zu '@label' suchen" href='@nwbib.routes.Application.search(subject=if(id.contains("/gnd/")) {id} else {"\""+label+"\""})'/>@label</a></span>
<span style="white-space: nowrap;">
<a title="Nach weiteren Titeln zu '@label' suchen" href='@nwbib.routes.Application.search(subject=if(id.contains("/gnd/")) {id} else {"\""+label+"\""})'/>@label</a>
@if(id.startsWith("http") && !id.contains("/sw/")){<a title="Linked-Data-Quelle zu '@label' abrufen" href='@id'><span class='glyphicon glyphicon-link'></span></a>}
</span>
@if(components.last == component) {
@for(topic <- (subject\"label").asOpt[String]; hits = Lobid.getTotalHits("subject.label.raw", topic.trim, CONFIG.getString("nwbib.filter")).get(Lobid.API_TIMEOUT); if hits > 1){
| <a href='@nwbib.routes.Application.search(q="subject.label.raw:\""+topic.trim+"\"")' title="Alle @hits Titel zum Thema '@topic.trim' anzeigen"><span class='badge progress-bar-success'>@hits</span></a>
Expand Down Expand Up @@ -234,9 +237,9 @@
}
}

@subordinateSearchFor(id: String) = {
@subordinate("isPartOf.hasSuperordinate.id", id, "Bände", ("zugehöriger Band", "zugehörige Bände"))
@subordinate("containedIn.id", id, "Enthält", ("Beitrag", "Beiträge"))
@subordinateSearchFor(ids: Seq[String]) = {
@subordinate("isPartOf.hasSuperordinate.id", ids, "Bände", ("zugehöriger Band", "zugehörige Bände"))
@subordinate("containedIn.id", ids, "Enthält", ("Beitrag", "Beiträge"))
}

@table(){
Expand Down Expand Up @@ -264,14 +267,17 @@
@labelled("In", "containedIn")

@part_of("isPartOf", "hasSuperordinate")
@subordinateSearchFor(String.format("http://lobid.org/resources/%s#!", (doc \ "hbzId").asOpt[String].getOrElse("")))
@subordinateSearchFor(String.format("https://lobid.org/resources/%s", (doc \ "rpbId").asOpt[String].getOrElse("")))
@subordinateSearchFor(String.format("http://lobid.org/resources/"+ZDB_PREFIX+"%s#!", (doc \ "zdbId").asOpt[String].getOrElse("")))
@subordinateSearchFor(Seq(
String.format("http://lobid.org/resources/"+ZDB_PREFIX+"%s#!", (doc \ "zdbId").asOpt[String].getOrElse("")),
String.format("http://lobid.org/resources/%s#!", (doc \ "hbzId").asOpt[String].getOrElse("")),
String.format("https://lobid.org/resources/%s", (doc \ "rpbId").asOpt[String].getOrElse("")),
String.format("http://rpb.lobid.org/%s", (doc \ "rpbId").asOpt[String].getOrElse(""))
))
@parallelausgabe()
@raumsystematik("https://rpb.lobid.org/spatial")
@sachsystematik("http://purl.org/lobid/rpb")

@subjects((doc \ "subject").asOpt[Seq[JsValue]].getOrElse(Seq()).filter(v => !(v \ "source").toString.matches(".*(Systematik|Notationen|Dewey).*")))
@subjects((doc \ "subject").asOpt[Seq[JsValue]].getOrElse(Seq()).filter(v => !(v \ "source").toString.matches(".*(Systematik|Notationen|Sachgruppen|Dewey).*")))

@result_field("Schlagwortfolge", "subjectChain", doc, TableRow.VALUES, valueLabel = Option(Seq()))

Expand Down
Loading

0 comments on commit 9a5b411

Please sign in to comment.