Skip to content

Commit

Permalink
Merge pull request #151 from scc-digitalhub/solr_roles
Browse files Browse the repository at this point in the history
Solr roles
  • Loading branch information
matteo-s authored Feb 24, 2025
2 parents 3d5a1f3 + 0d75a5c commit 0937caf
Show file tree
Hide file tree
Showing 6 changed files with 148 additions and 59 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package it.smartcommunitylabdhub.core.components.solr;

import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
import org.thymeleaf.TemplateEngine;
import org.thymeleaf.context.Context;

import it.smartcommunitylabdhub.core.models.indexers.SolrEntityIndexer;
import lombok.extern.slf4j.Slf4j;

@Component
@Slf4j
@Profile("generate-solr")
public class SolrFieldsExportRunner implements CommandLineRunner {
@Autowired
ApplicationContext context;

@Autowired
private TemplateEngine templateEngine;

private List<SolrEntityIndexer<?>> services;

@Autowired(required = false)
public void setServices(List<SolrEntityIndexer<?>> services) {
this.services = services;
}

@Override
public void run(String... args) throws Exception {
log.info("Running solr fields export...");
int returnCode = 0;
if(services != null) {
try {
Map<String, IndexField> fieldsMap = new HashMap<>();
services.forEach(service -> {
for(IndexField field : service.fields()) {
if(!fieldsMap.containsKey(field.getName()))
fieldsMap.put(field.getName(), field);
}
});
Map<String, Object> variables = new HashMap<>();
variables.put("fields", fieldsMap.values());
final Context ctx = new Context();
ctx.setVariables(variables);
String content = templateEngine.process("solr_fields.xml", ctx);
String out = "solr/solr_fields.xml";
Path fp = Paths.get(out);
Files.createDirectories(fp.getParent());
File file = new File(out);
log.info("writing solr fields to {}...",file.getAbsolutePath());
FileUtils.writeStringToFile(file, content, StandardCharsets.UTF_8);
} catch (Exception e) {
log.error("Error with export: {}", e.getMessage());
returnCode = 1;
}
}
int exitCode = returnCode == 0
? SpringApplication.exit(context, () -> 0)
: SpringApplication.exit(context, () -> 1);
System.exit(exitCode);
}
}
Original file line number Diff line number Diff line change
@@ -1,21 +1,15 @@
package it.smartcommunitylabdhub.core.components.solr;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import it.smartcommunitylabdhub.commons.jackson.JacksonMapper;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;

import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient.Builder;
Expand All @@ -31,6 +25,7 @@
import org.apache.solr.common.params.MultiMapSolrParams;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.util.Assert;
Expand All @@ -39,12 +34,20 @@
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;

import it.smartcommunitylabdhub.commons.jackson.JacksonMapper;
import lombok.extern.slf4j.Slf4j;

@Slf4j
public class SolrIndexManager {

private static final TypeReference<HashMap<String, Serializable>> typeRef = new TypeReference<
HashMap<String, Serializable>
>() {};
private static final TypeReference<HashMap<String, Serializable>> typeRef = new TypeReference<HashMap<String, Serializable>>() {};

private final SolrProperties props;
private final Http2SolrClient solrClient;
Expand All @@ -65,7 +68,7 @@ public SolrIndexManager(SolrProperties props) {
Builder builder = new Http2SolrClient.Builder(props.getUrl())
.withConnectionTimeout(props.getTimeout(), TimeUnit.MILLISECONDS);

if (StringUtils.hasLength(props.getUrl()) && StringUtils.hasLength(props.getPassword())) {
if (StringUtils.hasLength(props.getUser()) && StringUtils.hasLength(props.getPassword())) {
//add basic auth
builder.withBasicAuthCredentials(props.getUser(), props.getPassword());
}
Expand All @@ -79,36 +82,36 @@ public void init() throws SolrIndexerException {
//check if collection exists
String solrUrl = props.getUrl();
String baseUri = solrUrl.endsWith("/") ? solrUrl : solrUrl + "/";


HttpHeaders headers = new HttpHeaders();
if(StringUtils.hasLength(props.getAdminUser()) && StringUtils.hasLength(props.getAdminPassword())) {
String auth = props.getAdminUser() + ":" + props.getAdminPassword();
String authHeader = Base64.getEncoder().encodeToString(auth.getBytes());
headers.setBasicAuth(authHeader);
log.debug("init solr collection auth {}", authHeader);
}

try {
String listUrl = baseUri + "admin/collections?action=LIST";
ResponseEntity<String> listResponse = restTemplate.getForEntity(listUrl, String.class);
ResponseEntity<String> listResponse = restTemplate.exchange(listUrl, HttpMethod.GET, new HttpEntity<String>(headers), String.class);

if (listResponse.getStatusCode().isError()) {
throw new SolrIndexerException(
String.format(
"can not talk to solr {%s}: {%s}",
listResponse.getStatusCode().toString(),
listResponse.getBody()
)
);
log.warn("can not talk to solr {}: {}",
listResponse.getStatusCode().toString(),
listResponse.getBody());
}

initCollection();
initCollection(headers);
} catch (HttpClientErrorException e) {
//fallback to core if 400
//creation is NOT supported
String listUrl = baseUri + "admin/cores?action=STATUS";
ResponseEntity<String> listResponse = restTemplate.getForEntity(listUrl, String.class);
ResponseEntity<String> listResponse = restTemplate.exchange(listUrl, HttpMethod.GET, new HttpEntity<String>(headers), String.class);

if (listResponse.getStatusCode().isError()) {
throw new SolrIndexerException(
String.format(
"can not talk to solr {%s}: {%s}",
listResponse.getStatusCode().toString(),
listResponse.getBody()
)
);
log.warn("can not talk to solr {}: {}",
listResponse.getStatusCode().toString(),
listResponse.getBody());
}

Map<String, Serializable> map = JacksonMapper.OBJECT_MAPPER.readValue(listResponse.getBody(), typeRef);
Expand All @@ -118,7 +121,7 @@ public void init() throws SolrIndexerException {
}
}
} catch (SolrException | RestClientException | JsonProcessingException e) {
throw new SolrIndexerException(e.getMessage());
log.warn("can not initialize solr: {}", e.getMessage());
}
}

Expand All @@ -140,12 +143,21 @@ public synchronized void initFields(Iterable<IndexField> fields) throws SolrInde
if (log.isTraceEnabled()) {
log.trace("fields: {}", fields);
}

HttpHeaders headers = new HttpHeaders();
if(StringUtils.hasLength(props.getUser()) && StringUtils.hasLength(props.getPassword())) {
String auth = props.getUser() + ":" + props.getPassword();
String authHeader = Base64.getEncoder().encodeToString(auth.getBytes());
headers.setBasicAuth(authHeader);
log.debug("init solr fields auth {}", authHeader);
}

String solrUrl = props.getUrl();
String baseUri = solrUrl.endsWith("/") ? solrUrl : solrUrl + "/";
String fieldsUri = baseUri + props.getCollection() + "/schema/fields";
//check existing fields
ResponseEntity<String> responseEntity = restTemplate.getForEntity(fieldsUri, String.class);
ResponseEntity<String> responseEntity =restTemplate.exchange(fieldsUri, HttpMethod.GET, new HttpEntity<String>(headers), String.class);
//ResponseEntity<String> responseEntity = restTemplate.getForEntity(fieldsUri, String.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
String schemaUri = baseUri + props.getCollection() + "/schema";
try {
Expand All @@ -166,7 +178,8 @@ public synchronized void initFields(Iterable<IndexField> fields) throws SolrInde
field.isMultiValued(),
field.isStored(),
field.isUninvertible(),
schemaUri
schemaUri,
headers
);
}
}
Expand Down Expand Up @@ -319,7 +332,8 @@ private void addField(
boolean multiValued,
boolean stored,
boolean uninvertible,
String uri
String uri,
HttpHeaders headers
) throws SolrIndexerException {
ObjectNode rootNode = mapper.createObjectNode();
ObjectNode addNode = rootNode.putObject("add-field");
Expand All @@ -330,7 +344,7 @@ private void addField(
.put("indexed", indexed)
.put("stored", stored)
.put("uninvertible", uninvertible);
HttpEntity<ObjectNode> request = new HttpEntity<>(rootNode);
HttpEntity<ObjectNode> request = new HttpEntity<>(rootNode, headers);
ResponseEntity<String> response = restTemplate.exchange(uri, HttpMethod.POST, request, String.class);
if (response.getStatusCode().isError()) {
throw new SolrIndexerException(
Expand Down Expand Up @@ -403,21 +417,18 @@ private MultiMapSolrParams prepareQuery(
return new MultiMapSolrParams(queryParamMap);
}

private void initCollection() throws SolrIndexerException {
private void initCollection(HttpHeaders headers) throws SolrIndexerException {
try {
//check if collection exists
String solrUrl = props.getUrl();
String baseUri = solrUrl.endsWith("/") ? solrUrl : solrUrl + "/";
String listUrl = baseUri + "admin/collections?action=LIST";
ResponseEntity<String> listResponse = restTemplate.getForEntity(listUrl, String.class);
ResponseEntity<String> listResponse = restTemplate.exchange(listUrl, HttpMethod.GET, new HttpEntity<String>(headers), String.class);
if (listResponse.getStatusCode().isError()) {
throw new SolrIndexerException(
String.format(
"can not talk to solr {%s}: {%s}",
listResponse.getStatusCode().toString(),
listResponse.getBody()
)
);
log.warn("can not talk to solr {}: {}",
listResponse.getStatusCode().toString(),
listResponse.getBody());
return;
}

Map<String, Serializable> map = JacksonMapper.OBJECT_MAPPER.readValue(listResponse.getBody(), typeRef);
Expand All @@ -429,26 +440,20 @@ private void initCollection() throws SolrIndexerException {
String createUrl =
baseUri +
"admin/collections?action=CREATE&name={collection}&numShards={numShards}&replicationFactor={replicationFactor}&maxShardsPerNode=1";
ResponseEntity<String> createResponse = restTemplate.getForEntity(
createUrl,
String.class,
props.getCollection(),
props.getShards(),
props.getReplicas()
ResponseEntity<String> createResponse = restTemplate.exchange(createUrl, HttpMethod.GET, new HttpEntity<String>(headers), String.class,
props.getCollection(),
props.getShards(),
props.getReplicas()
);

if (createResponse.getStatusCode().isError()) {
throw new SolrIndexerException(
String.format(
"can not talk to solr {%s}: {%s}",
createResponse.getStatusCode().toString(),
createResponse.getBody()
)
);
log.warn("can not talk to solr {}: {}",
listResponse.getStatusCode().toString(),
listResponse.getBody());
}
}
} catch (SolrException | IOException e) {
throw new SolrIndexerException(e.getMessage());
log.warn("can not initialize solr: {}", e.getMessage());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ public class SolrProperties {
private String collection;
private String user;
private String password;
private String adminUser;
private String adminPassword;

private Integer timeout;
private Integer shards;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ protected SolrInputDocument parse(D item, String type) {
public List<IndexField> fields() {
List<IndexField> fields = new LinkedList<>();

fields.add(new IndexField("id", "string", true, false, true, true));
fields.add(new IndexField("keyGroup", "string", true, false, true, true));
fields.add(new IndexField("type", "string", true, false, true, true));

Expand Down
2 changes: 2 additions & 0 deletions application/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,8 @@ solr:
url: ${SOLR_URL:false}
user: ${SOLR_USER:}
password: ${SOLR_PASSWORD:}
admin-user: ${SOLR_ADMIN_USER:${SOLR_USER}}
admin-password: ${SOLR_ADMIN_PASSWORD:${SOLR_PASSWORD}}
collection: ${SOLR_COLLECTION:dhcore}
timeout: ${SOLR_TIMEOUT:5000}
shards: ${SOLR_COLLECTION_SHARDS_NUM:1}
Expand Down
3 changes: 3 additions & 0 deletions application/src/main/resources/templates/solr_fields.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
<root>
<field th:each="item : ${fields}" th:name="${item.name}" th:type="${item.type}" th:indexed="${item.indexed}" th:multiValued="${item.multiValued}" th:stored="${item.stored}" th:uninvertible="${item.uninvertible}"/>
</root>

0 comments on commit 0937caf

Please sign in to comment.