Skip to content

GitLab

Commit 17662de8 authored by Gradl, Tobias's avatar Gradl, Tobias
Browse files

422: Fix overlapping collection index stats (CLOSED) 

Task-Url: #422
parent 00412c53
Pipeline #23407 failed with stage
in 9 seconds
Hide whitespace changes
Inline Side-by-side
search-core/src/main/java/eu/dariah/de/search/config/LogConfigProperties.java
View file @ 17662de8
......@@ -13,4 +13,6 @@ public class LogConfigProperties {
private int maxHistory = 90;
private String totalSizeCap = "1GB";
private String maxFileSize = "10MB";
private boolean logQueries;
}
search-core/src/main/java/eu/dariah/de/search/es/client/SearchClientImpl.java
View file @ 17662de8
......@@ -8,13 +8,15 @@ import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import de.unibamberg.minf.core.util.Stopwatch;
import eu.dariah.de.search.config.LogConfigProperties;
import eu.dariah.de.search.es.client.base.BaseEsClientImpl;
import eu.dariah.de.search.es.service.params.GetParams;
import eu.dariah.de.search.es.service.params.SearchParams;
......@@ -23,8 +25,7 @@ import eu.dariah.de.search.query.results.NullSearchResponse;
@Component
public class SearchClientImpl extends BaseEsClientImpl implements SearchClient {
@Value("${debugging.querying.log_queries:#{false}}")
private boolean logQueries;
@Autowired private LogConfigProperties logConfig;
@Override
public GetResponse get(GetParams params) {
......@@ -47,21 +48,20 @@ public class SearchClientImpl extends BaseEsClientImpl implements SearchClient {
try {
CountResponse response;
Stopwatch swQuery = new Stopwatch().start();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
CountRequest countRequest = new CountRequest(params.getIndexNames()).source(searchSourceBuilder);
QueryBuilder queryBuilder;
if (params.getQuery()!=null) {
searchSourceBuilder.query(params.getQuery());
queryBuilder = params.getQuery();
} else {
searchSourceBuilder.query(QueryBuilders.matchAllQuery());
queryBuilder = QueryBuilders.matchAllQuery();
}
if (logQueries && logger.isInfoEnabled()) {
logger.info(searchSourceBuilder.toString());
CountRequest countRequest = new CountRequest(params.getIndexNames()).query(queryBuilder);
if (logConfig.isLogQueries() && logger.isInfoEnabled()) {
logger.info(queryBuilder.toString());
}
response = client.count(countRequest, RequestOptions.DEFAULT);
if (logQueries && logger.isDebugEnabled()) {
if (logConfig.isLogQueries() && logger.isDebugEnabled()) {
logger.debug("Query travel time: {}ms", swQuery.getElapsedTime());
}
return response.getCount();
......@@ -109,12 +109,12 @@ public class SearchClientImpl extends BaseEsClientImpl implements SearchClient {
searchSourceBuilder.highlighter(params.getHighlightBuilder());
}
if (logQueries && logger.isInfoEnabled()) {
if (logConfig.isLogQueries() && logger.isInfoEnabled()) {
logger.info(searchSourceBuilder.toString());
}
response = client.search(searchRequest, RequestOptions.DEFAULT);
if (logQueries && logger.isDebugEnabled()) {
if (logConfig.isLogQueries() && logger.isDebugEnabled()) {
logger.debug("Query travel time: {}ms, ES took: {}ms", swQuery.getElapsedTime(), response.getTook().getMillis());
}
......
search-core/src/main/java/eu/dariah/de/search/es/service/SearchServiceImpl.java
View file @ 17662de8
package eu.dariah.de.search.es.service;
import static org.springframework.util.Assert.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
......@@ -19,10 +21,9 @@ import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import de.unibamberg.minf.dme.model.base.Element;
import de.unibamberg.minf.dme.model.base.Nonterminal;
import de.unibamberg.minf.processing.model.base.Resource;
import eu.dariah.de.search.Constants.AggregationTypes;
import eu.dariah.de.search.Constants.QueryTypes;
......@@ -77,8 +78,8 @@ public class SearchServiceImpl implements SearchService {
@Override
public List<QueryBuilder> generateQuery(QueryTypes[] queryTypes, String[] paths, String[] expressions, Operator[] operators, ExtendedDatamodelContainer dmc) {
Assert.isTrue(queryTypes.length == paths.length);
Assert.isTrue(paths.length == expressions.length);
isTrue(queryTypes.length == paths.length, () -> "Arrays for query types and paths must be equal in length");
isTrue(paths.length == expressions.length, () -> "Arrays for paths and expressions must be equal in length");
Element root = dmc.getOrRenderProcessingRoot(false);
......@@ -98,7 +99,7 @@ public class SearchServiceImpl implements SearchService {
@Override
public AggregationBuilder generateAggregation(AggregationTypes aggregationType, String path, String aggregationName, int size, ExtendedDatamodelContainer dmc, List<AggregationBuilder> immediateAggs) {
List<AggregationBuilder> builders = this.generateAggregation(new AggregationTypes[] {aggregationType}, new String[] {path}, new String[] {aggregationName}, new int[] {size}, dmc, immediateAggs);
if (builders==null || builders.size()==0) {
if (builders==null || builders.isEmpty()) {
return null;
}
return builders.get(0);
......@@ -106,8 +107,8 @@ public class SearchServiceImpl implements SearchService {
@Override
public List<AggregationBuilder> generateAggregation(AggregationTypes[] aggregationTypes, String[] paths, String[] aggregationNames, int[] sizes, ExtendedDatamodelContainer dmc, List<AggregationBuilder> immediateAggs) {
Assert.isTrue(aggregationTypes.length == paths.length);
Assert.isTrue(paths.length == aggregationNames.length);
isTrue(aggregationTypes.length == paths.length, () -> "Arrays for aggregation types and paths must be equal in length");
isTrue(paths.length == aggregationNames.length, () -> "Arrays for paths and aggregation names must be equal in length");
Element root = dmc.getOrRenderProcessingRoot(false);
......@@ -125,8 +126,8 @@ public class SearchServiceImpl implements SearchService {
}
private List<QueryBuilder> createQueries(QueryTypes[] queryTypes, String[] paths, String[][] pathArrays, int depth, Object[] expressions, Operator[] operators, Element element) {
Map<String, List<Integer>> keyIndicesMap = new HashMap<String, List<Integer>>();
List<QueryBuilder> queryList = new ArrayList<QueryBuilder>();
Map<String, List<Integer>> keyIndicesMap = new HashMap<>();
List<QueryBuilder> queryList = new ArrayList<>();
for (int i=0; i<pathArrays.length; i++) {
try {
......@@ -175,12 +176,12 @@ public class SearchServiceImpl implements SearchService {
}
if (ExtendedElement.class.isAssignableFrom(element.getClass()) && ((ExtendedElement)element).isNested() && queryList.size()>0) {
String nestedPath = "";
StringBuilder nestedPath = new StringBuilder();
for (int i=0; i<depth; i++) {
// Any path is ok as they are the same up to this depth
nestedPath = nestedPath + pathArrays[0][i];
nestedPath.append(pathArrays[0][i]);
if (i<depth-1) {
nestedPath = nestedPath + ".";
nestedPath.append(".");
}
}
......@@ -194,9 +195,9 @@ public class SearchServiceImpl implements SearchService {
cq = queryList.get(0);
}
NestedQueryBuilder nq = QueryBuilders.nestedQuery(nestedPath, cq, ScoreMode.Avg);
NestedQueryBuilder nq = QueryBuilders.nestedQuery(nestedPath.toString(), cq, ScoreMode.Avg);
queryList = new ArrayList<QueryBuilder>();
queryList = new ArrayList<>();
queryList.add(nq);
......@@ -213,13 +214,13 @@ public class SearchServiceImpl implements SearchService {
} else if (queryType.equals(QueryTypes.QueryStringQuery)) {
return QueryBuilders.queryStringQuery(value.toString()).field(path).defaultOperator(operator);
} else {
throw new QueryExecutionException(String.format("Invalid QueryType: not implemented {}", queryType.toString()));
throw new QueryExecutionException(String.format("Invalid QueryType: not implemented %s", queryType.toString()));
}
}
private List<AggregationBuilder> createAggregations(AggregationTypes[] aggregationTypes, String[] paths, String[][] pathArrays, int depth, String[] aggregationNames, int[] sizes, Element element, List<AggregationBuilder> immediateAggs) {
Map<String, List<Integer>> keyIndicesMap = new HashMap<String, List<Integer>>();
List<AggregationBuilder> aggregationList = new ArrayList<AggregationBuilder>();
Map<String, List<Integer>> keyIndicesMap = new HashMap<>();
List<AggregationBuilder> aggregationList = new ArrayList<>();
for (int i=0; i<pathArrays.length; i++) {
if (pathArrays[i]==null) {
......@@ -308,7 +309,7 @@ public class SearchServiceImpl implements SearchService {
} else if (aggregationType.equals(AggregationTypes.SignificantTermsAggregation)) {
return AggregationBuilders.significantTerms(name).field(path).size(size);
} else {
throw new QueryExecutionException(String.format("Invalid AggregationType: not implemented {}", aggregationType.toString()));
throw new QueryExecutionException(String.format("Invalid AggregationType: not implemented %s", aggregationType.toString()));
}
}
}
\ No newline at end of file
search-ui/src/main/resources/application.yml
View file @ 17662de8
......@@ -32,7 +32,7 @@ log:
# pattern: "%-4relative [%thread] %-5level %logger{35}[%line]- %msg%n"
indexing:
baseDir: /tmp/indexing_logs
#log_queries: false
indexing:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment