diff --git a/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrFieldsExportRunner.java b/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrFieldsExportRunner.java new file mode 100644 index 00000000..65193f66 --- /dev/null +++ b/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrFieldsExportRunner.java @@ -0,0 +1,76 @@ +package it.smartcommunitylabdhub.core.components.solr; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.SpringApplication; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; +import org.thymeleaf.TemplateEngine; +import org.thymeleaf.context.Context; + +import it.smartcommunitylabdhub.core.models.indexers.SolrEntityIndexer; +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +@Profile("generate-solr") +public class SolrFieldsExportRunner implements CommandLineRunner { + @Autowired + ApplicationContext context; + + @Autowired + private TemplateEngine templateEngine; + + private List> services; + + @Autowired(required = false) + public void setServices(List> services) { + this.services = services; + } + + @Override + public void run(String... args) throws Exception { + log.info("Running solr fields export..."); + int returnCode = 0; + if(services != null) { + try { + Map fieldsMap = new HashMap<>(); + services.forEach(service -> { + for(IndexField field : service.fields()) { + if(!fieldsMap.containsKey(field.getName())) + fieldsMap.put(field.getName(), field); + } + }); + Map variables = new HashMap<>(); + variables.put("fields", fieldsMap.values()); + final Context ctx = new Context(); + ctx.setVariables(variables); + String content = templateEngine.process("solr_fields.xml", ctx); + String out = "solr/solr_fields.xml"; + Path fp = Paths.get(out); + Files.createDirectories(fp.getParent()); + File file = new File(out); + log.info("writing solr fields to {}...",file.getAbsolutePath()); + FileUtils.writeStringToFile(file, content, StandardCharsets.UTF_8); + } catch (Exception e) { + log.error("Error with export: {}", e.getMessage()); + returnCode = 1; + } + } + int exitCode = returnCode == 0 + ? SpringApplication.exit(context, () -> 0) + : SpringApplication.exit(context, () -> 1); + System.exit(exitCode); + } +} diff --git a/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrIndexManager.java b/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrIndexManager.java index 4be7f395..824d7286 100644 --- a/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrIndexManager.java +++ b/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrIndexManager.java @@ -1,21 +1,15 @@ package it.smartcommunitylabdhub.core.components.solr; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import it.smartcommunitylabdhub.commons.jackson.JacksonMapper; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; + import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.impl.Http2SolrClient.Builder; @@ -31,6 +25,7 @@ import org.apache.solr.common.params.MultiMapSolrParams; import org.springframework.data.domain.Pageable; import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.ResponseEntity; import org.springframework.util.Assert; @@ -39,12 +34,20 @@ import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import it.smartcommunitylabdhub.commons.jackson.JacksonMapper; +import lombok.extern.slf4j.Slf4j; + @Slf4j public class SolrIndexManager { - private static final TypeReference> typeRef = new TypeReference< - HashMap - >() {}; + private static final TypeReference> typeRef = new TypeReference>() {}; private final SolrProperties props; private final Http2SolrClient solrClient; @@ -65,7 +68,7 @@ public SolrIndexManager(SolrProperties props) { Builder builder = new Http2SolrClient.Builder(props.getUrl()) .withConnectionTimeout(props.getTimeout(), TimeUnit.MILLISECONDS); - if (StringUtils.hasLength(props.getUrl()) && StringUtils.hasLength(props.getPassword())) { + if (StringUtils.hasLength(props.getUser()) && StringUtils.hasLength(props.getPassword())) { //add basic auth builder.withBasicAuthCredentials(props.getUser(), props.getPassword()); } @@ -79,36 +82,36 @@ public void init() throws SolrIndexerException { //check if collection exists String solrUrl = props.getUrl(); String baseUri = solrUrl.endsWith("/") ? solrUrl : solrUrl + "/"; - + + HttpHeaders headers = new HttpHeaders(); + if(StringUtils.hasLength(props.getAdminUser()) && StringUtils.hasLength(props.getAdminPassword())) { + String auth = props.getAdminUser() + ":" + props.getAdminPassword(); + String authHeader = Base64.getEncoder().encodeToString(auth.getBytes()); + headers.setBasicAuth(authHeader); + log.debug("init solr collection auth {}", authHeader); + } + try { String listUrl = baseUri + "admin/collections?action=LIST"; - ResponseEntity listResponse = restTemplate.getForEntity(listUrl, String.class); + ResponseEntity listResponse = restTemplate.exchange(listUrl, HttpMethod.GET, new HttpEntity(headers), String.class); if (listResponse.getStatusCode().isError()) { - throw new SolrIndexerException( - String.format( - "can not talk to solr {%s}: {%s}", - listResponse.getStatusCode().toString(), - listResponse.getBody() - ) - ); + log.warn("can not talk to solr {}: {}", + listResponse.getStatusCode().toString(), + listResponse.getBody()); } - initCollection(); + initCollection(headers); } catch (HttpClientErrorException e) { //fallback to core if 400 //creation is NOT supported String listUrl = baseUri + "admin/cores?action=STATUS"; - ResponseEntity listResponse = restTemplate.getForEntity(listUrl, String.class); + ResponseEntity listResponse = restTemplate.exchange(listUrl, HttpMethod.GET, new HttpEntity(headers), String.class); if (listResponse.getStatusCode().isError()) { - throw new SolrIndexerException( - String.format( - "can not talk to solr {%s}: {%s}", - listResponse.getStatusCode().toString(), - listResponse.getBody() - ) - ); + log.warn("can not talk to solr {}: {}", + listResponse.getStatusCode().toString(), + listResponse.getBody()); } Map map = JacksonMapper.OBJECT_MAPPER.readValue(listResponse.getBody(), typeRef); @@ -118,7 +121,7 @@ public void init() throws SolrIndexerException { } } } catch (SolrException | RestClientException | JsonProcessingException e) { - throw new SolrIndexerException(e.getMessage()); + log.warn("can not initialize solr: {}", e.getMessage()); } } @@ -140,12 +143,21 @@ public synchronized void initFields(Iterable fields) throws SolrInde if (log.isTraceEnabled()) { log.trace("fields: {}", fields); } + + HttpHeaders headers = new HttpHeaders(); + if(StringUtils.hasLength(props.getUser()) && StringUtils.hasLength(props.getPassword())) { + String auth = props.getUser() + ":" + props.getPassword(); + String authHeader = Base64.getEncoder().encodeToString(auth.getBytes()); + headers.setBasicAuth(authHeader); + log.debug("init solr fields auth {}", authHeader); + } String solrUrl = props.getUrl(); String baseUri = solrUrl.endsWith("/") ? solrUrl : solrUrl + "/"; String fieldsUri = baseUri + props.getCollection() + "/schema/fields"; //check existing fields - ResponseEntity responseEntity = restTemplate.getForEntity(fieldsUri, String.class); + ResponseEntity responseEntity =restTemplate.exchange(fieldsUri, HttpMethod.GET, new HttpEntity(headers), String.class); + //ResponseEntity responseEntity = restTemplate.getForEntity(fieldsUri, String.class); if (responseEntity.getStatusCode().is2xxSuccessful()) { String schemaUri = baseUri + props.getCollection() + "/schema"; try { @@ -166,7 +178,8 @@ public synchronized void initFields(Iterable fields) throws SolrInde field.isMultiValued(), field.isStored(), field.isUninvertible(), - schemaUri + schemaUri, + headers ); } } @@ -319,7 +332,8 @@ private void addField( boolean multiValued, boolean stored, boolean uninvertible, - String uri + String uri, + HttpHeaders headers ) throws SolrIndexerException { ObjectNode rootNode = mapper.createObjectNode(); ObjectNode addNode = rootNode.putObject("add-field"); @@ -330,7 +344,7 @@ private void addField( .put("indexed", indexed) .put("stored", stored) .put("uninvertible", uninvertible); - HttpEntity request = new HttpEntity<>(rootNode); + HttpEntity request = new HttpEntity<>(rootNode, headers); ResponseEntity response = restTemplate.exchange(uri, HttpMethod.POST, request, String.class); if (response.getStatusCode().isError()) { throw new SolrIndexerException( @@ -403,21 +417,18 @@ private MultiMapSolrParams prepareQuery( return new MultiMapSolrParams(queryParamMap); } - private void initCollection() throws SolrIndexerException { + private void initCollection(HttpHeaders headers) throws SolrIndexerException { try { //check if collection exists String solrUrl = props.getUrl(); String baseUri = solrUrl.endsWith("/") ? solrUrl : solrUrl + "/"; String listUrl = baseUri + "admin/collections?action=LIST"; - ResponseEntity listResponse = restTemplate.getForEntity(listUrl, String.class); + ResponseEntity listResponse = restTemplate.exchange(listUrl, HttpMethod.GET, new HttpEntity(headers), String.class); if (listResponse.getStatusCode().isError()) { - throw new SolrIndexerException( - String.format( - "can not talk to solr {%s}: {%s}", - listResponse.getStatusCode().toString(), - listResponse.getBody() - ) - ); + log.warn("can not talk to solr {}: {}", + listResponse.getStatusCode().toString(), + listResponse.getBody()); + return; } Map map = JacksonMapper.OBJECT_MAPPER.readValue(listResponse.getBody(), typeRef); @@ -429,26 +440,20 @@ private void initCollection() throws SolrIndexerException { String createUrl = baseUri + "admin/collections?action=CREATE&name={collection}&numShards={numShards}&replicationFactor={replicationFactor}&maxShardsPerNode=1"; - ResponseEntity createResponse = restTemplate.getForEntity( - createUrl, - String.class, - props.getCollection(), - props.getShards(), - props.getReplicas() + ResponseEntity createResponse = restTemplate.exchange(createUrl, HttpMethod.GET, new HttpEntity(headers), String.class, + props.getCollection(), + props.getShards(), + props.getReplicas() ); - + if (createResponse.getStatusCode().isError()) { - throw new SolrIndexerException( - String.format( - "can not talk to solr {%s}: {%s}", - createResponse.getStatusCode().toString(), - createResponse.getBody() - ) - ); + log.warn("can not talk to solr {}: {}", + listResponse.getStatusCode().toString(), + listResponse.getBody()); } } } catch (SolrException | IOException e) { - throw new SolrIndexerException(e.getMessage()); + log.warn("can not initialize solr: {}", e.getMessage()); } } } diff --git a/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrProperties.java b/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrProperties.java index 66b1f2a3..40a74cc3 100644 --- a/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrProperties.java +++ b/application/src/main/java/it/smartcommunitylabdhub/core/components/solr/SolrProperties.java @@ -17,6 +17,8 @@ public class SolrProperties { private String collection; private String user; private String password; + private String adminUser; + private String adminPassword; private Integer timeout; private Integer shards; diff --git a/application/src/main/java/it/smartcommunitylabdhub/core/models/indexers/BaseEntityIndexer.java b/application/src/main/java/it/smartcommunitylabdhub/core/models/indexers/BaseEntityIndexer.java index 7c8584f6..feda2eed 100644 --- a/application/src/main/java/it/smartcommunitylabdhub/core/models/indexers/BaseEntityIndexer.java +++ b/application/src/main/java/it/smartcommunitylabdhub/core/models/indexers/BaseEntityIndexer.java @@ -84,6 +84,7 @@ protected SolrInputDocument parse(D item, String type) { public List fields() { List fields = new LinkedList<>(); + fields.add(new IndexField("id", "string", true, false, true, true)); fields.add(new IndexField("keyGroup", "string", true, false, true, true)); fields.add(new IndexField("type", "string", true, false, true, true)); diff --git a/application/src/main/resources/application.yml b/application/src/main/resources/application.yml index 83c7db9b..90e31940 100644 --- a/application/src/main/resources/application.yml +++ b/application/src/main/resources/application.yml @@ -211,6 +211,8 @@ solr: url: ${SOLR_URL:false} user: ${SOLR_USER:} password: ${SOLR_PASSWORD:} + admin-user: ${SOLR_ADMIN_USER:${SOLR_USER}} + admin-password: ${SOLR_ADMIN_PASSWORD:${SOLR_PASSWORD}} collection: ${SOLR_COLLECTION:dhcore} timeout: ${SOLR_TIMEOUT:5000} shards: ${SOLR_COLLECTION_SHARDS_NUM:1} diff --git a/application/src/main/resources/templates/solr_fields.xml b/application/src/main/resources/templates/solr_fields.xml new file mode 100644 index 00000000..903a84aa --- /dev/null +++ b/application/src/main/resources/templates/solr_fields.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file