Skip to content

Commit

Permalink
Ingester: process the links of the simplification documents independe…
Browse files Browse the repository at this point in the history 
…ntly of the links of the full process documents
  • Loading branch information
@josegar74
josegar74 committed Dec 19, 2024
1 parent 3f34d7b commit abc6350
Show file tree
Hide file tree
Showing 2 changed files with 122 additions and 17 deletions.
82 changes: 71 additions & 11 deletions ...ces/ingester/src/main/java/com/geocat/ingester/model/linkchecker/helper/DocumentLink.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
import javax.persistence.Transient;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;

//Represents a link in a document
@MappedSuperclass
Expand Down Expand Up @@ -84,22 +86,52 @@ public DocumentLink() {

//--

public static List<String> validProtocols = Arrays.asList(new String[]{
public static List<String> validViewProtocols = Arrays.asList(new String[]{
"wms",
"http://www.opengis.net/def/serviceType/ogc/wms".toLowerCase(),
"OGC Web Map Service".toLowerCase(),
"Web Map Service (WMS)".toLowerCase(),
"OGC:WMS".toLowerCase(),
"http://www.opengeospatial.org/standards/wms",
"wmts",
"http://www.opengis.net/def/serviceType/ogc/wmts".toLowerCase(),
"OGC Web Map Tile Service".toLowerCase(),
"OGC:WMTS".toLowerCase(),
"http://www.opengeospatial.org/standards/wmts"
});

public static List<String> validDownloadProtocols = Arrays.asList(new String[]{
"wfs",
"http://www.opengis.net/def/serviceType/ogc/wfs".toLowerCase(),
"OGC Web Feature Service".toLowerCase(),
"Web Feature Service (WFS)".toLowerCase(),
"OGC:WFS".toLowerCase(),
"http://www.opengeospatial.org/standards/wfs",
"atom",
"https://tools.ietf.org/html/rfc4287".toLowerCase(),
"ATOM Syndication Format".toLowerCase(),
"INSPIRE Atom".toLowerCase(),
"wcs",
"OGC:WCS".toLowerCase(),
"http://www.opengis.net/def/serviceType/ogc/wcs".toLowerCase(),
"api features",
"OGC - API Features".toLowerCase(),
"OGC:OGC-API-Features-items".toLowerCase(),
"HTTP:OGC:API-Features".toLowerCase(),
"http://www.opengis.net/def/interface/ogcapi-features".toLowerCase(),
"SensorThings".toLowerCase(),
"sos",
"OGC:SOS".toLowerCase(),
"http://www.opengis.net/def/serviceType/ogc/sos".toLowerCase()
});

public static List<String> validProtocols = Stream.concat(validViewProtocols.stream(),
validDownloadProtocols.stream()).collect(Collectors.toList());

public static List<String> validAtomProtocols = Arrays.asList(new String[]{
"https://tools.ietf.org/html/rfc4287".toLowerCase(),
"ATOM Syndication Format".toLowerCase(),
"OGC Web Feature Service".toLowerCase(),
"OGC Web Map Service".toLowerCase(),
"OGC Web Map Tile Service".toLowerCase(),
"wms",
"wmts",
"wfs",
"atom",
"http://www.opengeospatial.org/standards/wms",
"http://www.opengeospatial.org/standards/wmts",
"http://www.opengeospatial.org/standards/wfs",
"INSPIRE Atom".toLowerCase()
});

Expand All @@ -110,10 +142,37 @@ public DocumentLink() {
"http://inspire.ec.europa.eu/metadata-codelist/SpatialDataServiceType/view".toLowerCase()
});

public static final String VALID_PROTOCOLS_VIEW_REGEX = "(.*wms.*|.*wmts.*|.*web map service.*)";

public static final String VALID_PROTOCOLS_DOWNLOAD_REGEX = "(.*wfs.*|.*atom.*|.*wcs.*|.*sos.*|.*api.*feature.*|.*sensorthings.*|.*web feature service.*)";

public static final String VALID_PROTOCOLS_REGEX = "(.*wfs.*|.*atom.*|.*wcs.*|.*sos.*|.*api.*feature.*|.*sensorthings.*|.*wms.*|.*wmts.*|.*web map service.*|.*web feature service.*)";

public boolean isInspireSimplifiedLink() {
// Relax the check to process links with the applicationProfile information
if ((rawURL == null) || (protocol == null))
return false;
if (rawURL.isEmpty() || protocol.isEmpty())
return false;

if (!validProtocols.contains(protocol.toLowerCase())) {
// Check protocol match "simple" values instead of exact match
if (!protocol.toLowerCase().matches(VALID_PROTOCOLS_REGEX)) {
return false;
}
}


return true;
}


/*public boolean isInspireSimplifiedLink() {
if ((rawURL == null) || (protocol == null) || (applicationProfile == null))
if ((rawURL == null) || (protocol == null))
return false;
if (rawURL.isEmpty() || protocol.isEmpty() || applicationProfile.isEmpty())
if (rawURL.isEmpty() || protocol.isEmpty())
return false;
if (!validProtocols.contains(protocol.toLowerCase()))
Expand All @@ -123,7 +182,8 @@ public boolean isInspireSimplifiedLink() {
return false;
return true;
}
}*/



//--
Expand Down
57 changes: 51 additions & 6 deletions services/ingester/src/main/java/com/geocat/ingester/service/IngesterService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,10 @@ public class IngesterService {
@Autowired
DatasetDocumentLinkRepo datasetDocumentLinkRepo;

@Autowired
LocalDatasetMetadataRecordRepo localDatasetMetadataRecordRepo;


@Autowired
LinkToDataRepo linkToDataRepo;

Expand Down Expand Up @@ -402,9 +406,26 @@ private void processViewLinksIndicators(MetadataRecordXml metadata, List<LinkToD

LinkedResourceIndicator indicator = new LinkedResourceIndicator(layerName, layerLink, associatedServiceIds);
viewServiceLinkedResourceIndicators.add(indicator);

}
});


if (viewLinks.isEmpty()) {
LocalDatasetMetadataRecord datasetMetadataRecord = localDatasetMetadataRecordRepo.findFirstByFileIdentifierAndLinkCheckJobId(metadata.getRecordIdentifier(), linkCheckJobId);
List<DocumentLink> viewLinksMetadataOnlineResources = datasetMetadataRecord.getDocumentLinks().stream()
.filter(x -> (x.getLinkState().equals(LinkState.Complete) && x.getLinkHTTPStatusCode() != null && x.getLinkHTTPStatusCode() == 200) && (DocumentLink.validViewProtocols.contains(x.getProtocol().toLowerCase()) || x.getProtocol().toLowerCase().matches(DocumentLink.VALID_PROTOCOLS_VIEW_REGEX)))
.collect(Collectors.toList());

for (DocumentLink documentLink : viewLinksMetadataOnlineResources) {
List<String> associatedServiceIds = new ArrayList<>();
associatedServiceIds.add(datasetMetadataRecord.getFileIdentifier());

LinkedResourceIndicator indicator = new LinkedResourceIndicator(documentLink.getRawURL(), documentLink.getRawURL(), associatedServiceIds);
viewServiceLinkedResourceIndicators.add(indicator);
}
}

ObjectMapper objectMapper = new ObjectMapper();
try {
addOrUpdateIndicator(metadata, "INDICATOR_VIEW_SERVICE_LINKED_RESOURCES",
Expand Down Expand Up @@ -447,16 +468,25 @@ private void processDownloadLinksIndicators(MetadataRecordXml metadata, List<Lin

// TODO: Review
List<AtomActualDataEntry> atomActualDataEntryList = simpleAtomLinkToData.getAtomActualDataEntryList();
for (AtomActualDataEntry atomActualDataEntry : atomActualDataEntryList) {
if ((atomActualDataEntry.getSuccessfullyDownloaded() != null) && atomActualDataEntry.getSuccessfullyDownloaded()) {
for (AtomDataRequest atomDataRequest : atomActualDataEntry.getAtomDataRequestList()) {
layerLink = atomDataRequest.getFinalURL();
if (atomActualDataEntryList.isEmpty()) {
if (simpleAtomLinkToData.getAtomSubFeedRequest() != null) {
layerLink = simpleAtomLinkToData.getAtomSubFeedRequest().getFinalURL();
}

if (!StringUtils.hasLength(layerLink)) {
layerLink = StringUtils.hasLength(simpleAtomLinkToData.getLayerId()) ? simpleAtomLinkToData.getLayerId() : "";
}
} else {
for (AtomActualDataEntry atomActualDataEntry : atomActualDataEntryList) {
if ((atomActualDataEntry.getSuccessfullyDownloaded() != null) && atomActualDataEntry.getSuccessfullyDownloaded()) {
for (AtomDataRequest atomDataRequest : atomActualDataEntry.getAtomDataRequestList()) {
layerLink = atomDataRequest.getFinalURL();
break;
}
break;
}
break;
}
}

}

List<ServiceDocumentLink> serviceDocumentLinks = serviceDocumentLinkRepo.findByLinkCheckJobIdAndSha2(linkCheckJobId, capabilitiesSha2);
Expand All @@ -472,6 +502,21 @@ private void processDownloadLinksIndicators(MetadataRecordXml metadata, List<Lin
downloadServiceLinkedResourceIndicators.add(indicator);
});

if (downloadLinks.isEmpty()) {
LocalDatasetMetadataRecord datasetMetadataRecord = localDatasetMetadataRecordRepo.findFirstByFileIdentifierAndLinkCheckJobId(metadata.getRecordIdentifier(), linkCheckJobId);
List<DocumentLink> downloadLinksMetadataOnlineResources = datasetMetadataRecord.getDocumentLinks().stream()
.filter(x -> (x.getLinkState().equals(LinkState.Complete) && x.getLinkHTTPStatusCode() != null && x.getLinkHTTPStatusCode() == 200) && (DocumentLink.validDownloadProtocols.contains(x.getProtocol().toLowerCase()) || x.getProtocol().toLowerCase().matches(DocumentLink.VALID_PROTOCOLS_DOWNLOAD_REGEX)))
.collect(Collectors.toList());

for (DocumentLink documentLink : downloadLinksMetadataOnlineResources) {
List<String> associatedServiceIds = new ArrayList<>();
associatedServiceIds.add(datasetMetadataRecord.getFileIdentifier());

LinkedResourceIndicator indicator = new LinkedResourceIndicator(documentLink.getRawURL(), documentLink.getRawURL(), associatedServiceIds);
downloadServiceLinkedResourceIndicators.add(indicator);
}
}

ObjectMapper objectMapper = new ObjectMapper();
try {
addOrUpdateIndicator(metadata, "INDICATOR_DOWNLOAD_SERVICE_LINKED_RESOURCES",
Expand Down

0 comments on commit abc6350

Please sign in to comment.