Merge pull request #19 from gdi-be/cleanup-terms-handling

Adapt import and export to use terms of use codelist

mde-importer/src/main/java/de/terrestris/mde/mde_importer/importer/ImportService.java

@@ -24,6 +24,7 @@
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
 import java.io.IOException;
+import java.math.BigInteger;
 import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -35,6 +36,7 @@
 import java.util.stream.Stream;
 
 import static de.terrestris.mde.mde_backend.model.json.JsonIsoMetadata.InspireTheme.*;
+import static de.terrestris.mde.mde_backend.service.IsoGenerator.TERMS_OF_USE_MAP;
 import static de.terrestris.utils.xml.MetadataNamespaceUtils.XLINK;
 import static de.terrestris.utils.xml.XmlUtils.*;
 
@@ -50,6 +52,18 @@ public class ImportService {
 
   public static final Map<String, JsonIsoMetadata.InspireTheme> INSPIRE_THEME_MAP;
 
+  private static final Set<String> AUTO_KEYWORDS = Set.of(
+    "inspireidentifiziert",
+    "open data",
+    "opendata",
+    "Sachdaten",
+    "Karten",
+    "Geodaten",
+    "Berlin",
+    "infoFeatureAccessService",
+    "infoMapAccessService"
+  );
+
   static {
     FORMAT.setTimeZone(TimeZone.getTimeZone("UTC"));
     INSPIRE_THEME_MAP = new HashMap<>();
@@ -200,6 +214,10 @@ private void parseDatasetMetadata(XMLStreamReader reader) throws XMLStreamExcept
     if (list != null) {
       list.forEach(file -> addService(file, json, client.getData(), technical.getData()));
     }
+    if (json.getTermsOfUseId() == null) {
+      log.info("Terms of use could not be mapped for {}, using standard.", metadata.getMetadataId());
+      json.setTermsOfUseId(BigInteger.ONE);
+    }
     isoMetadataRepository.save(metadata);
     clientMetadataRepository.save(client);
     technicalMetadataRepository.save(technical);
@@ -220,7 +238,6 @@ private static void extractCoordinateSystem(XMLStreamReader reader, JsonIsoMetad
   }
 
   private static void extractFromIso(XMLStreamReader reader, IsoMetadata metadata, JsonIsoMetadata json, ClientMetadata client, TechnicalMetadata technical) throws XMLStreamException, ParseException {
-    json.setResourceConstraints(new ArrayList<>());
     skipToElement(reader, "MD_DataIdentification");
     metadata.setMetadataId(reader.getAttributeValue(null, "uuid"));
     client.setMetadataId(metadata.getMetadataId());
@@ -445,29 +462,18 @@ private static void extractTransferOptions(XMLStreamReader reader, JsonIsoMetada
 
   private static void extractResourceConstraints(XMLStreamReader reader, JsonIsoMetadata json) throws XMLStreamException {
     if (reader.isStartElement() && reader.getLocalName().equals("resourceConstraints")) {
-      var list = new ArrayList<Constraint>();
-      json.getResourceConstraints().add(list);
       while (!(reader.isEndElement() && reader.getLocalName().equals("resourceConstraints"))) {
         reader.next();
         if (!reader.isStartElement() || reader.getLocalName().equals("MD_LegalConstraints") ||
           reader.getLocalName().equals("useLimitation") || reader.getLocalName().equals("CharacterString")) {
           continue;
         }
-        var constraint = new Constraint();
-        list.add(constraint);
-        constraint.setType(Constraint.ConstraintType.valueOf(reader.getLocalName()));
         skipToOneOf(reader, "Anchor", "CharacterString", "MD_RestrictionCode");
-        switch (reader.getLocalName()) {
-          case "MD_RestrictionCode":
-            constraint.setRestrictionCode(MD_RestrictionCode.valueOf(reader.getAttributeValue(null, "codeListValue")));
-            break;
-          case "Anchor":
-            constraint.setNamespace(reader.getAttributeValue(XLINK, "href"));
-            constraint.setText(reader.getElementText());
-            break;
-          case "CharacterString":
-            constraint.setText(reader.getElementText());
-            break;
+        if (reader.getLocalName().equals("CharacterString")) {
+          String text = reader.getElementText();
+          if (TERMS_OF_USE_MAP.get(text) != null) {
+            json.setTermsOfUseId(BigInteger.valueOf(TERMS_OF_USE_MAP.get(text).getId()));
+          }
         }
       }
     }
@@ -486,12 +492,18 @@ private static void extractKeyword(XMLStreamReader reader, JsonIsoMetadata json)
           case "keyword":
             skipToOneOf(reader, "CharacterString", "Anchor");
             if (reader.getLocalName().equals("CharacterString")) {
-              keywords.add(new Keyword(null, reader.getElementText()));
+              var keyword = reader.getElementText();
+              if (!AUTO_KEYWORDS.contains(keyword)) {
+                keywords.add(new Keyword(null, keyword));
+              }
             } else {
               var keyword = new Keyword();
               keyword.setNamespace(reader.getAttributeValue(XLINK, "href"));
-              keyword.setKeyword(reader.getElementText());
-              keywords.add(keyword);
+              var text = reader.getElementText();
+              keyword.setKeyword(text);
+              if (!AUTO_KEYWORDS.contains(text)) {
+                keywords.add(keyword);
+              }
             }
             break;
           case "thesaurusName":
@@ -511,6 +523,9 @@ private static void extractKeyword(XMLStreamReader reader, JsonIsoMetadata json)
             break;
         }
       }
+      if (keywords.isEmpty()) {
+        return;
+      }
       json.getKeywords().put(thesaurus.getTitle() == null ? "default" : thesaurus.getTitle(), keywords);
       json.getThesauri().put(thesaurus.getTitle() == null ? "default" : thesaurus.getTitle(), thesaurus);
       if (thesaurus.getTitle() != null && thesaurus.getTitle().equals("GEMET - INSPIRE themes, version 1.0")) {

mde-services/src/main/java/de/terrestris/mde/mde_backend/model/json/JsonIsoMetadata.java

@@ -110,8 +110,6 @@ public enum InspireTheme {
 
   private List<ContentDescription> contentDescriptions;
 
-  private List<List<Constraint>> resourceConstraints = new ArrayList<>();
-
   private String lineage;
 
   private boolean valid;

mde-services/src/main/java/de/terrestris/mde/mde_backend/model/json/termsofuse/Json.java

@@ -0,0 +1,24 @@
+package de.terrestris.mde.mde_backend.model.json.termsofuse;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@JsonDeserialize(as = Json.class)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@NoArgsConstructor(force = true)
+@AllArgsConstructor
+public class Json {
+
+  private String id;
+
+  private String name;
+
+  private String url;
+
+  private String quelle;
+
+}

mde-services/src/main/java/de/terrestris/mde/mde_backend/model/json/termsofuse/TermsOfUse.java

@@ -0,0 +1,34 @@
+package de.terrestris.mde.mde_backend.model.json.termsofuse;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@JsonDeserialize(as = TermsOfUse.class)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@NoArgsConstructor(force = true)
+@AllArgsConstructor
+public class TermsOfUse {
+
+  private int id;
+
+  private String shortname;
+
+  private boolean active;
+
+  private String description;
+
+  private List<String> matchStrings;
+
+  private boolean openData;
+
+  private Json json;
+
+  private String note;
+
+}

mde-services/src/main/java/de/terrestris/mde/mde_backend/service/DatasetIsoGenerator.java

@@ -1,11 +1,12 @@
 package de.terrestris.mde.mde_backend.service;
 
-import de.terrestris.mde.mde_backend.enumeration.MetadataProfile;
-import de.terrestris.mde.mde_backend.model.json.Constraint;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import de.terrestris.mde.mde_backend.model.json.Extent;
 import de.terrestris.mde.mde_backend.model.json.JsonIsoMetadata;
 import de.terrestris.mde.mde_backend.model.json.Source;
 import de.terrestris.mde.mde_backend.model.json.codelists.MD_MaintenanceFrequencyCode;
+import de.terrestris.mde.mde_backend.model.json.termsofuse.TermsOfUse;
 import lombok.extern.log4j.Log4j2;
 import org.codehaus.stax2.XMLOutputFactory2;
 import org.springframework.stereotype.Component;
@@ -19,9 +20,12 @@
 import java.time.format.DateTimeFormatter;
 import java.util.List;
 
+import static de.terrestris.mde.mde_backend.enumeration.MetadataProfile.ISO;
 import static de.terrestris.mde.mde_backend.model.json.codelists.CI_DateTypeCode.*;
+import static de.terrestris.mde.mde_backend.model.json.codelists.MD_RestrictionCode.otherRestrictions;
 import static de.terrestris.mde.mde_backend.model.json.codelists.MD_ScopeCode.dataset;
 import static de.terrestris.mde.mde_backend.service.GeneratorUtils.*;
+import static de.terrestris.mde.mde_backend.service.IsoGenerator.TERMS_OF_USE_BY_ID;
 import static de.terrestris.mde.mde_backend.service.IsoGenerator.replaceValues;
 import static de.terrestris.utils.xml.MetadataNamespaceUtils.*;
 import static de.terrestris.utils.xml.XmlUtils.writeSimpleElement;
@@ -30,6 +34,8 @@
 @Log4j2
 public class DatasetIsoGenerator {
 
+  private static final ObjectMapper MAPPER = new ObjectMapper();
+
   private static final XMLOutputFactory FACTORY = XMLOutputFactory2.newFactory();
 
   private static void writeIdentifier(XMLStreamWriter writer, String identifier) throws XMLStreamException {
@@ -64,27 +70,33 @@ protected static void writePreviews(XMLStreamWriter writer, List<Source> preview
     }
   }
 
-  protected static void writeResourceConstraints(XMLStreamWriter writer, List<List<Constraint>> constraints) throws XMLStreamException {
-    for (var list : constraints) {
-      writer.writeStartElement(GMD, "resourceConstraints");
-      writer.writeStartElement(GMD, "MD_LegalConstraints");
-      for (var constraint : list) {
-        writer.writeStartElement(GMD, constraint.getType().toString());
-        if (constraint.getRestrictionCode() != null) {
-          writeCodelistValue(writer, constraint.getRestrictionCode());
-        } else if (constraint.getNamespace() != null) {
-          writer.writeStartElement(GMX, "Anchor");
-          writer.writeAttribute(XLINK, "href", constraint.getNamespace());
-          writer.writeCharacters(constraint.getText());
-          writer.writeEndElement(); // Anchor
-        } else {
-          writeSimpleElement(writer, GCO, "CharacterString", constraint.getText());
-        }
-        writer.writeEndElement();
-      }
-      writer.writeEndElement(); // MD_LegalConstraints
-      writer.writeEndElement(); // resourceConstraints
-    }
+  protected static void writeResourceConstraints(XMLStreamWriter writer, TermsOfUse terms) throws XMLStreamException, JsonProcessingException {
+    writer.writeStartElement(GMD, "resourceConstraints");
+    writer.writeStartElement(GMD, "MD_LegalConstraints");
+    writer.writeStartElement(GMD, "accessConstraints");
+    writeCodelistValue(writer, otherRestrictions);
+    writer.writeEndElement(); // accessConstraints
+    writer.writeStartElement(GMD, "otherConstraints");
+    writer.writeStartElement(GMX, "Anchor");
+    writer.writeAttribute(XLINK, "href", "http://inspire.ec.europa.eu/metadata-codelist/LimitationsOnPublicAccess/noLimitations");
+    writer.writeCharacters("Es gelten keine Zugriffsbeschränkungen");
+    writer.writeEndElement(); // Anchor
+    writer.writeEndElement(); // otherConstraints
+    writer.writeEndElement(); // MD_LegalConstraints
+    writer.writeEndElement(); // resourceConstraints
+    writer.writeStartElement(GMD, "resourceConstraints");
+    writer.writeStartElement(GMD, "MD_LegalConstraints");
+    writer.writeStartElement(GMD, "useConstraints");
+    writeCodelistValue(writer, otherRestrictions);
+    writer.writeEndElement(); // useConstraints
+    writer.writeStartElement(GMD, "otherConstraints");
+    writeSimpleElement(writer, GCO, "CharacterString", terms.getDescription());
+    writer.writeEndElement(); // otherConstraints
+    writer.writeStartElement(GMD, "otherConstraints");
+    writeSimpleElement(writer, GCO, "CharacterString", MAPPER.writeValueAsString(terms.getJson()));
+    writer.writeEndElement(); // otherConstraints
+    writer.writeEndElement(); // MD_LegalConstraints
+    writer.writeEndElement(); // resourceConstraints
   }
 
   private static void writeSpatialResolution(XMLStreamWriter writer, JsonIsoMetadata metadata) throws XMLStreamException {
@@ -165,7 +177,7 @@ protected static void writeExtent(XMLStreamWriter writer, Extent extent, String
     writer.writeEndElement(); // extent
   }
 
-  protected static void writeIdentificationInfo(XMLStreamWriter writer, JsonIsoMetadata metadata, String id) throws XMLStreamException {
+  protected static void writeIdentificationInfo(XMLStreamWriter writer, JsonIsoMetadata metadata, String id) throws XMLStreamException, JsonProcessingException {
     writer.writeStartElement(GMD, "identificationInfo");
     writer.writeStartElement(GMD, "MD_DataIdentification");
     writer.writeAttribute("uuid", id);
@@ -189,7 +201,7 @@ protected static void writeIdentificationInfo(XMLStreamWriter writer, JsonIsoMet
     writeMaintenanceInfo(writer, metadata.getMaintenanceFrequency());
     writePreviews(writer, metadata.getPreviews());
     writeKeywords(writer, metadata);
-    writeResourceConstraints(writer, metadata.getResourceConstraints());
+    writeResourceConstraints(writer, TERMS_OF_USE_BY_ID.get(metadata.getTermsOfUseId().intValue()));
     writeSpatialResolution(writer, metadata);
     writeLanguage(writer);
     writeCharacterSet(writer);
@@ -199,11 +211,22 @@ protected static void writeIdentificationInfo(XMLStreamWriter writer, JsonIsoMet
     writer.writeEndElement(); // identificationInfo
   }
 
+  private static void writeKeyword(XMLStreamWriter writer, String keyword) throws XMLStreamException {
+    writer.writeStartElement(GMD, "keyword");
+    writeSimpleElement(writer, GCO, "CharacterString", keyword);
+    writer.writeEndElement(); // keyword
+  }
+
   protected static void writeKeywords(XMLStreamWriter writer, JsonIsoMetadata metadata) throws XMLStreamException {
     for (var entry : metadata.getKeywords().entrySet()) {
       var thesaurus = metadata.getThesauri().get(entry.getKey());
       writer.writeStartElement(GMD, "descriptiveKeywords");
       writer.writeStartElement(GMD, "MD_Keywords");
+      if (entry.getKey().equals("default")) {
+        if (!metadata.getMetadataProfile().equals(ISO)) {
+          writeKeyword(writer, "inspireidentifiziert");
+        }
+      }
       for (var keyword : entry.getValue()) {
         writer.writeStartElement(GMD, "keyword");
         if (keyword.getNamespace() != null) {
@@ -294,7 +317,7 @@ protected static void writeDataQualityInfo(XMLStreamWriter writer, JsonIsoMetada
     writer.writeEndElement(); // level
     writer.writeEndElement(); // DQ_Scope
     writer.writeEndElement(); // scope
-    if (!metadata.getMetadataProfile().equals(MetadataProfile.ISO)) {
+    if (!metadata.getMetadataProfile().equals(ISO)) {
       writeReport(writer, metadata, service);
     }
     writer.writeStartElement(GMD, "lineage");