Commit 10627869 authored by Gradl, Tobias's avatar Gradl, Tobias
Browse files

1326: Implement strategy to increase performance when metaquerying

Task-Url: https://pm.winseda.de/issues/1326
parent 98bf3267
Pipeline #11918 failed with stages
in 7 seconds
......@@ -30,7 +30,6 @@ import eu.dariah.de.search.query.Query;
import eu.dariah.de.search.query.SimpleQuery;
import eu.dariah.de.search.query.SimpleQueryImpl;
import eu.dariah.de.search.query.execution.AggregationService;
import eu.dariah.de.search.query.execution.QueryExecutionService;
import eu.dariah.de.search.query.execution.QueryExecutionServiceImpl;
import eu.dariah.de.search.query.meta.SruQueryExecutionServiceImpl;
import eu.dariah.de.search.query.results.QueryResult;
......@@ -48,11 +47,10 @@ public class QueryExecutionController {
@Autowired private AggregationService aggregationService;
@Autowired private CustomSearchService customSearchService;
@Autowired protected DatamodelService datamodelService;
@Autowired private ObjectMapper objMapper;
@Autowired protected ObjectMapper objMapper;
@GetMapping(value={"/",""})
@GetMapping
public @ResponseBody List<QueryResult> queryAsGet(@RequestParam String q, HttpServletResponse response, Locale locale) {
SimpleQuery query = new SimpleQueryImpl();
query.setQueryString(q);
......@@ -62,15 +60,11 @@ public class QueryExecutionController {
if (qr!=null) {
results.add(qr);
}
qr = sruQueryExecutionService.executeQuery(query, locale);
if (qr!=null) {
results.add(qr);
}
results.addAll(sruQueryExecutionService.prepareQuery(query, locale));
return results;
}
@PostMapping(value={"/",""})
@PostMapping
public @ResponseBody List<QueryResult> queryAsPost(@RequestBody String q, HttpServletResponse response, Locale locale) throws QueryExecutionException {
Query query = this.deserializeQuery(q);
List<QueryResult> results = new ArrayList<>();
......@@ -89,11 +83,8 @@ public class QueryExecutionController {
if (qr!=null) {
results.add(qr);
}
results.addAll(sruQueryExecutionService.prepareQuery(query, locale));
qr = sruQueryExecutionService.executeQuery(query, locale);
if (qr!=null) {
results.add(qr);
}
return results;
}
......@@ -111,7 +102,6 @@ public class QueryExecutionController {
return result;
}
// We handle this explicitly mainly for API access
private Query deserializeQuery(String jsonQuery) throws QueryExecutionException {
Query query = null;
try {
......
......@@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import de.unibamberg.minf.processing.model.SerializableResource;
import de.unibamberg.minf.processing.model.serialization.ResourceDeserializer;
import eu.dariah.de.search.model.Filter;
import eu.dariah.de.search.query.results.QueryResult.SearchTypes;
public abstract class BaseQuery implements Query {
......@@ -19,6 +20,7 @@ public abstract class BaseQuery implements Query {
private String customSearch;
private List<String> tagAggregators;
private List<Filter> filters;
private List<SearchTypes> resultTypes;
@JsonDeserialize(contentUsing=ResourceDeserializer.class)
private List<SerializableResource> entities;
......@@ -53,4 +55,7 @@ public abstract class BaseQuery implements Query {
public List<Filter> getFilters() { return filters; }
public void setFilters(List<Filter> filters) { this.filters = filters; }
public List<SearchTypes> getResultTypes() { return resultTypes; }
public void setResultTypes(List<SearchTypes> resultTypes) { this.resultTypes = resultTypes; }
}
......@@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import de.unibamberg.minf.processing.model.SerializableResource;
import eu.dariah.de.search.model.Filter;
import eu.dariah.de.search.query.results.QueryResult.SearchTypes;
@JsonIgnoreProperties(ignoreUnknown = true)
public interface Query {
......@@ -38,4 +39,7 @@ public interface Query {
public List<Filter> getFilters();
public void setFilters(List<Filter> filters);
public List<SearchTypes> getResultTypes();
public void setResultTypes(List<SearchTypes> resultTypes);
}
......@@ -12,7 +12,6 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.elasticsearch.index.query.SimpleQueryStringBuilder;
import org.elasticsearch.search.SearchHit;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -24,8 +23,6 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.MissingNode;
import de.unibamberg.minf.core.util.json.JsonNodeHelper;
import de.unibamberg.minf.processing.model.SerializableResource;
import de.unibamberg.minf.processing.model.base.Resource;
......@@ -75,7 +72,7 @@ public class ItemServiceImpl extends BaseResultService implements ItemService {
}
GetResponse response = searchService.get(params);
if (response.isExists()) {
return objMapper.readTree(response.getSourceAsString());
return resourceAwareObjectMapper.readTree(response.getSourceAsString());
} else {
return null;
}
......@@ -127,7 +124,7 @@ public class ItemServiceImpl extends BaseResultService implements ItemService {
private void renderReferencingResultElement(ResultElement reReferenced, SearchHit hit, Map<String, List<Resource>> itemIdTitleMap, String referenceId, Locale locale) {
try {
JsonNode nHit = objMapper.readTree(hit.getSourceAsString());
JsonNode nHit = resourceAwareObjectMapper.readTree(hit.getSourceAsString());
JsonNode nReferences = jsonNodeHelper.findOneRecursive(nHit, Constants.ELEMENT_KEY_REFERENCES);
JsonNode nRelatedId;
......@@ -239,7 +236,7 @@ public class ItemServiceImpl extends BaseResultService implements ItemService {
JsonNode nId;
for (SearchHit hit : resp.getHits().getHits()) {
try {
nHit = objMapper.readTree(hit.getSourceAsString());
nHit = resourceAwareObjectMapper.readTree(hit.getSourceAsString());
nId = jsonNodeHelper.findOneRecursive(nHit, Constants.ELEMENT_KEY_ID);
relatedRe = this.renderResultElement(nHit, null, hit.getIndex(), hit.getId(), locale);
......
......@@ -36,7 +36,7 @@ import eu.dariah.de.search.query.execution.base.BaseQueryService;
import eu.dariah.de.search.query.results.QueryResult;
import eu.dariah.de.search.query.results.QueryResultDatasource;
import eu.dariah.de.search.query.results.ResultElement;
import eu.dariah.de.search.query.results.QueryResult.ResultTypes;
import eu.dariah.de.search.query.results.QueryResult.SearchTypes;
@Component
public class QueryExecutionServiceImpl extends BaseQueryService implements QueryExecutionService {
......@@ -57,7 +57,7 @@ public class QueryExecutionServiceImpl extends BaseQueryService implements Query
QueryResult qr = this.buildQueryResult(q, responses, locale);
qr.setExecutedQueries(this.renderExecutedQueries(queryIndexMap));
qr.setExecuted(true);
return qr;
}
......@@ -158,7 +158,7 @@ public class QueryExecutionServiceImpl extends BaseQueryService implements Query
protected QueryResult buildQueryResult(Query q, List<SearchResponse> responses, Locale locale) {
QueryResult qr = new QueryResult();
qr.setResultType(ResultTypes.REGULAR);
qr.setResultType(SearchTypes.REGULAR);
qr.setResultElements(new ArrayList<ResultElement>());
for (SearchResponse resp : responses) {
......@@ -208,7 +208,7 @@ public class QueryExecutionServiceImpl extends BaseQueryService implements Query
}
try {
o = objMapper.readTree(hit.getSourceAsString());
o = resourceAwareObjectMapper.readTree(hit.getSourceAsString());
sourceId = jsonNodeHelper.findOneRecursive(o, Constants.ELEMENT_KEY_COLLECTION_ID).asText();
......
......@@ -19,6 +19,7 @@ import org.springframework.util.Assert;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.fasterxml.jackson.databind.node.TextNode;
......@@ -26,10 +27,12 @@ import com.fasterxml.jackson.databind.node.TextNode;
import de.unibamberg.minf.core.util.json.JsonNodeHelper;
import de.unibamberg.minf.core.web.localization.MessageSource;
import de.unibamberg.minf.processing.model.base.Resource;
import de.unibamberg.minf.processing.model.serialization.ResourceSerializer;
import eu.dariah.de.search.Constants;
import eu.dariah.de.search.Constants.RootElementKeys;
import eu.dariah.de.search.model.Collection;
import eu.dariah.de.search.query.results.FieldHighlight;
import eu.dariah.de.search.query.results.QueryResultDatasource;
import eu.dariah.de.search.query.results.ResultElement;
import eu.dariah.de.search.service.CollectionService;
......@@ -37,7 +40,8 @@ public abstract class BaseResultService implements InitializingBean {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
@Autowired protected CollectionService collectionService;
@Autowired protected ObjectMapper objMapper;
protected ObjectMapper resourceAwareObjectMapper;
@Autowired private JsonNodeHelper jsonNodeHelper;
@Autowired private MessageSource messageSource;
......@@ -70,6 +74,12 @@ public abstract class BaseResultService implements InitializingBean {
@Override
public void afterPropertiesSet() throws Exception {
this.integrationsMessageCodes = this.isResolveMessageCodes(integrationModelEntityId);
resourceAwareObjectMapper = new ObjectMapper();
SimpleModule module = new SimpleModule();
module.addSerializer(Resource.class, new ResourceSerializer());
resourceAwareObjectMapper.registerModule(module);
}
protected ResultElement renderResultElement(JsonNode source, Collection collection, String index, String id, Locale locale) {
......@@ -77,6 +87,19 @@ public abstract class BaseResultService implements InitializingBean {
}
protected ResultElement renderResultElement(JsonNode source, Collection collection, String index, String id, Locale locale, float score, Explanation explanation, Map<String, String[]> fieldHighlights, boolean includeOriginal, boolean includeHtml) {
QueryResultDatasource qrd;
if (collection==null ) {
qrd = null;
} else {
qrd = new QueryResultDatasource();
qrd.setProviderName(collection.getName(locale.getISO3Language()));
qrd.setProviderId(collection.getId());
qrd.setImageUrl(collection.getImageUrl());
}
return this.renderResultElement(source, qrd, index, id, locale, score, explanation, fieldHighlights, includeOriginal, includeHtml);
}
protected ResultElement renderResultElement(JsonNode source, QueryResultDatasource datasource, String index, String id, Locale locale, float score, Explanation explanation, Map<String, String[]> fieldHighlights, boolean includeOriginal, boolean includeHtml) {
Assert.notNull(source);
String modelId = index.substring(indexNamePrefix.length());
......@@ -90,7 +113,6 @@ public abstract class BaseResultService implements InitializingBean {
qre.setExplanation(explanation);
qre.setScore(score);
try {
qre.setEndpointId(jsonNodeHelper.findOneRecursive(source, Constants.ELEMENT_KEY_ENDPOINT_ID).asText());
} catch (Exception e1) {
......@@ -98,11 +120,11 @@ public abstract class BaseResultService implements InitializingBean {
}
/* Collection */
if (collection!=null) {
qre.setCollectionId(collection.getId());
qre.setCollectionName(collection.getNames().values().iterator().next());
if (collection.getImageUrl()!=null) {
jsonNodeHelper.appendRecursive(source, Constants.PRESENTATION_PLACEHOLDER_IMAGE_SUBELEMENT_KEY.split("\\."), collection.getImageUrl());
if (datasource!=null) {
qre.setCollectionId(datasource.getProviderId());
qre.setCollectionName(datasource.getProviderName());
if (datasource.getImageUrl()!=null) {
jsonNodeHelper.appendRecursive(source, Constants.PRESENTATION_PLACEHOLDER_IMAGE_SUBELEMENT_KEY.split("\\."), datasource.getImageUrl());
}
}
......
......@@ -12,26 +12,16 @@ import org.apache.commons.io.IOUtils;
import org.apache.http.client.utils.URIBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import de.unibamberg.minf.dme.model.base.Element;
import de.unibamberg.minf.dme.model.base.Nonterminal;
import de.unibamberg.minf.dme.model.datamodel.base.Datamodel;
import de.unibamberg.minf.dme.model.mapping.base.Mapping;
import de.unibamberg.minf.processing.consumption.CollectingResourceConsumptionServiceImpl;
import de.unibamberg.minf.processing.model.base.Resource;
import de.unibamberg.minf.processing.model.serialization.ResourceSerializer;
import de.unibamberg.minf.processing.service.base.BaseResourceProcessingServiceImpl;
import de.unibamberg.minf.processing.service.xml.XmlProcessingService;
import eu.dariah.de.search.es.service.IndexingService;
import eu.dariah.de.search.indexing.model.ResourceContainer;
......@@ -41,19 +31,17 @@ import eu.dariah.de.search.model.Endpoint;
import eu.dariah.de.search.model.ExtendedDatamodelContainer;
import eu.dariah.de.search.query.Query;
import eu.dariah.de.search.query.SimpleQuery;
import eu.dariah.de.search.query.execution.QueryExecutionService;
import eu.dariah.de.search.query.execution.base.BaseResultService;
import eu.dariah.de.search.query.results.QueryResult;
import eu.dariah.de.search.query.results.ResultElement;
import eu.dariah.de.search.query.results.QueryResult.ResultTypes;
import eu.dariah.de.search.query.results.QueryResult.SearchTypes;
import eu.dariah.de.search.query.results.QueryResultDatasource;
import eu.dariah.de.search.service.CollectionService;
import eu.dariah.de.search.service.DatamodelService;
import eu.dariah.de.search.service.ResourceIndexingService;
import eu.dariah.de.search.service.MappingService;
import eu.dariah.de.search.transformation.ResourceEnrichmentService;
@Component
public class SruQueryExecutionServiceImpl extends BaseResultService implements QueryExecutionService {
public class SruQueryExecutionServiceImpl extends BaseResultService implements InitializingBean {
protected final Logger logger = LoggerFactory.getLogger(SruQueryExecutionServiceImpl.class);
......@@ -74,46 +62,32 @@ public class SruQueryExecutionServiceImpl extends BaseResultService implements Q
private String presentationModelEntityId;
@Override
public QueryResult executeQuery(Query q, Locale locale) {
List<Collection> coll;
if (q.getSourceIds()!=null && !q.getSourceIds().isEmpty()) {
coll = collectionService.findByIds(q.getSourceIds());
} else {
coll = collectionService.getAll();
}
public List<QueryResult> prepareQuery(Query query, Locale locale) {
List<QueryResult> result = new ArrayList<>();
Map<QueryResultDatasource, Endpoint> sruCollectionEndpointMap = this.collectSruSourceEndpointMap(query, locale);
ObjectMapper indexingObjectMapper = new ObjectMapper();
SimpleModule module = new SimpleModule();
module.addSerializer(Resource.class, new ResourceSerializer());
indexingObjectMapper.registerModule(module);
QueryResult prepResult;
for (QueryResultDatasource sruDatasource : sruCollectionEndpointMap.keySet()) {
prepResult = new QueryResult();
prepResult.setResultDatasources(new ArrayList<>());
prepResult.getResultDatasources().add(sruDatasource);
}
return result;
}
public List<QueryResult> executeQuery(Query q, Locale locale) {
List<QueryResult> result = new ArrayList<>();
Map<QueryResultDatasource, Endpoint> sruCollectionEndpointMap = this.collectSruSourceEndpointMap(q, locale);
String entityId;
String alias;
// Collect endpoints to federate
Endpoint opacEndpoint;
for (Collection c : coll) {
opacEndpoint = null;
boolean hasCrawlEp = false;
for (Endpoint e : c.getEndpoints()) {
if (e.getMethod().equals("OPAC")) {
opacEndpoint = e;
} else {
hasCrawlEp = true;
break;
}
}
if (hasCrawlEp || opacEndpoint==null) {
continue;
}
//String result = restTemplate.getForObject(opacEndpoint.getUrl(), String.class);
String entityId = null;
String alias = null;
for (Entry<QueryResultDatasource, Endpoint> sruEntry : sruCollectionEndpointMap.entrySet()) {
entityId = null;
alias = null;
for (Dataset ds : opacEndpoint.getDatasets()) {
for (Dataset ds : sruEntry.getValue().getDatasets()) {
entityId = ds.getId();
alias = ds.getRemoteAlias();
break;
......@@ -121,12 +95,6 @@ public class SruQueryExecutionServiceImpl extends BaseResultService implements Q
ExtendedDatamodelContainer s = datamodelService.findById(entityId);
/*if (s==null) {
Mapping m = mappingService.g(entityId);
s = datamodelService.findSchemaById(m.getSourceId());
}*/
Element r = s.getOrRenderElementHierarchy();
......@@ -134,7 +102,7 @@ public class SruQueryExecutionServiceImpl extends BaseResultService implements Q
ResourceEnrichmentService enrichmentService = appContext.getBean(ResourceEnrichmentService.class);
IndexingService indexingService = appContext.getBean(IndexingService.class);
indexingService.init(s.getIndexName(), s.getId(), c.getId(), opacEndpoint.getId());
indexingService.init(s.getIndexName(), s.getId(), sruEntry.getKey().getProviderId(), sruEntry.getValue().getId());
processingSvc.setSchema(s.getModel());
processingSvc.addConsumptionService(enrichmentService);
......@@ -145,7 +113,7 @@ public class SruQueryExecutionServiceImpl extends BaseResultService implements Q
processingSvc.setRoot((Nonterminal)r);
processingSvc.init();
URIBuilder b = new URIBuilder(opacEndpoint.getUrl());
URIBuilder b = new URIBuilder(sruEntry.getValue().getUrl());
b.addParameter("query", ((SimpleQuery)q).getQueryString());
b.addParameter("recordSchema", alias);
b.addParameter("maximumRecords", "10");
......@@ -162,7 +130,7 @@ public class SruQueryExecutionServiceImpl extends BaseResultService implements Q
QueryResult qr = new QueryResult();
qr.setResultType(ResultTypes.META);
qr.setResultType(SearchTypes.META);
qr.setResultElements(new ArrayList<ResultElement>());
if (!enrichmentService.getResourceBatch().isEmpty()) {
......@@ -173,19 +141,46 @@ public class SruQueryExecutionServiceImpl extends BaseResultService implements Q
for (ResourceContainer rc : enrichmentService.getResourceBatch()) {
qr.getResultElements().add(this.renderResultElement(indexingObjectMapper.convertValue(rc.toSource(), JsonNode.class), c, s.getIndexName(), rc.getId(), locale, 1, null, null, true, false));
qr.getResultElements().add(this.renderResultElement(resourceAwareObjectMapper.convertValue(rc.toSource(), JsonNode.class), sruEntry.getKey(), s.getIndexName(), rc.getId(), locale, 1, null, null, true, false));
}
}
return qr;
qr.setExecuted(true);
result.add(qr);
} catch (Exception e) {
logger.error("Error", e);
}
}
return null;
return result;
}
private Map<QueryResultDatasource, Endpoint> collectSruSourceEndpointMap(Query q, Locale locale) {
List<Collection> coll;
if (q.getSourceIds()!=null && !q.getSourceIds().isEmpty()) {
coll = collectionService.findByIds(q.getSourceIds());
} else {
coll = collectionService.getAll();
}
Map<QueryResultDatasource, Endpoint> sruSourceEndpointMap = new HashMap<>();
QueryResultDatasource qrd;
for (Collection c : coll) {
for (Endpoint e : c.getEndpoints()) {
if (e.getMethod().equals("OPAC")) {
qrd = new QueryResultDatasource();
qrd.setProviderName(c.getName(locale.getISO3Language()));
qrd.setProviderId(c.getId());
qrd.setImageUrl(c.getImageUrl());
sruSourceEndpointMap.put(qrd, e);
break;
}
}
}
return sruSourceEndpointMap;
}
......
......@@ -8,9 +8,9 @@ import eu.dariah.de.search.pojo.FilterPojo;
public class QueryResult {
public enum ResultTypes { REGULAR, META }
public enum SearchTypes { REGULAR, META }
private ResultTypes resultType;
private SearchTypes resultType;
private List<ResultElement> resultElements;
private List<QueryResultDatasource> resultDatasources;
private List<FilterPojo> availableFilters;
......@@ -23,10 +23,11 @@ public class QueryResult {
private int startIndex;
private boolean hasMore;
private List<String> executedQueries;
private boolean executed;
public ResultTypes getResultType() { return resultType; }
public void setResultType(ResultTypes resultType) { this.resultType = resultType; }
public SearchTypes getResultType() { return resultType; }
public void setResultType(SearchTypes resultType) { this.resultType = resultType; }
public List<FilterPojo> getAvailableFilters() { return availableFilters; }
public void setAvailableFilters(List<FilterPojo> availableFilters) { this.availableFilters = availableFilters; }
......@@ -63,4 +64,7 @@ public class QueryResult {
public List<String> getExecutedQueries() { return executedQueries; }
public void setExecutedQueries(List<String> executedQueries) { this.executedQueries = executedQueries; }
public boolean isExecuted() { return executed; }
public void setExecuted(boolean executed) { this.executed = executed; }
}
\ No newline at end of file
......@@ -3,6 +3,7 @@ package eu.dariah.de.search.query.results;
public class QueryResultDatasource implements Comparable<QueryResultDatasource> {
private String providerId;
private String providerName;
private String imageUrl;
private long hitCount;
private boolean loadTags;
......@@ -12,6 +13,9 @@ public class QueryResultDatasource implements Comparable<QueryResultDatasource>
public String getProviderName() { return providerName; }
public void setProviderName(String providerName) { this.providerName = providerName; }
public String getImageUrl() { return imageUrl; }
public void setImageUrl(String imageUrl) { this.imageUrl = imageUrl; }
public long getHitCount() { return hitCount; }
public void setHitCount(long hitCount) { this.hitCount = hitCount; }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment