Commit 17662de8 authored by Gradl, Tobias's avatar Gradl, Tobias
Browse files

422: Fix overlapping collection index stats (CLOSED)

Task-Url: #422
parent 00412c53
Pipeline #23407 failed with stage
in 9 seconds
...@@ -13,4 +13,6 @@ public class LogConfigProperties { ...@@ -13,4 +13,6 @@ public class LogConfigProperties {
private int maxHistory = 90; private int maxHistory = 90;
private String totalSizeCap = "1GB"; private String totalSizeCap = "1GB";
private String maxFileSize = "10MB"; private String maxFileSize = "10MB";
private boolean logQueries;
} }
...@@ -8,13 +8,15 @@ import org.elasticsearch.action.search.SearchType; ...@@ -8,13 +8,15 @@ import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import de.unibamberg.minf.core.util.Stopwatch; import de.unibamberg.minf.core.util.Stopwatch;
import eu.dariah.de.search.config.LogConfigProperties;
import eu.dariah.de.search.es.client.base.BaseEsClientImpl; import eu.dariah.de.search.es.client.base.BaseEsClientImpl;
import eu.dariah.de.search.es.service.params.GetParams; import eu.dariah.de.search.es.service.params.GetParams;
import eu.dariah.de.search.es.service.params.SearchParams; import eu.dariah.de.search.es.service.params.SearchParams;
...@@ -23,8 +25,7 @@ import eu.dariah.de.search.query.results.NullSearchResponse; ...@@ -23,8 +25,7 @@ import eu.dariah.de.search.query.results.NullSearchResponse;
@Component @Component
public class SearchClientImpl extends BaseEsClientImpl implements SearchClient { public class SearchClientImpl extends BaseEsClientImpl implements SearchClient {
@Value("${debugging.querying.log_queries:#{false}}") @Autowired private LogConfigProperties logConfig;
private boolean logQueries;
@Override @Override
public GetResponse get(GetParams params) { public GetResponse get(GetParams params) {
...@@ -47,21 +48,20 @@ public class SearchClientImpl extends BaseEsClientImpl implements SearchClient { ...@@ -47,21 +48,20 @@ public class SearchClientImpl extends BaseEsClientImpl implements SearchClient {
try { try {
CountResponse response; CountResponse response;
Stopwatch swQuery = new Stopwatch().start(); Stopwatch swQuery = new Stopwatch().start();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); QueryBuilder queryBuilder;
CountRequest countRequest = new CountRequest(params.getIndexNames()).source(searchSourceBuilder);
if (params.getQuery()!=null) { if (params.getQuery()!=null) {
searchSourceBuilder.query(params.getQuery()); queryBuilder = params.getQuery();
} else { } else {
searchSourceBuilder.query(QueryBuilders.matchAllQuery()); queryBuilder = QueryBuilders.matchAllQuery();
} }
if (logQueries && logger.isInfoEnabled()) { CountRequest countRequest = new CountRequest(params.getIndexNames()).query(queryBuilder);
logger.info(searchSourceBuilder.toString()); if (logConfig.isLogQueries() && logger.isInfoEnabled()) {
logger.info(queryBuilder.toString());
} }
response = client.count(countRequest, RequestOptions.DEFAULT); response = client.count(countRequest, RequestOptions.DEFAULT);
if (logQueries && logger.isDebugEnabled()) { if (logConfig.isLogQueries() && logger.isDebugEnabled()) {
logger.debug("Query travel time: {}ms", swQuery.getElapsedTime()); logger.debug("Query travel time: {}ms", swQuery.getElapsedTime());
} }
return response.getCount(); return response.getCount();
...@@ -109,12 +109,12 @@ public class SearchClientImpl extends BaseEsClientImpl implements SearchClient { ...@@ -109,12 +109,12 @@ public class SearchClientImpl extends BaseEsClientImpl implements SearchClient {
searchSourceBuilder.highlighter(params.getHighlightBuilder()); searchSourceBuilder.highlighter(params.getHighlightBuilder());
} }
if (logQueries && logger.isInfoEnabled()) { if (logConfig.isLogQueries() && logger.isInfoEnabled()) {
logger.info(searchSourceBuilder.toString()); logger.info(searchSourceBuilder.toString());
} }
response = client.search(searchRequest, RequestOptions.DEFAULT); response = client.search(searchRequest, RequestOptions.DEFAULT);
if (logQueries && logger.isDebugEnabled()) { if (logConfig.isLogQueries() && logger.isDebugEnabled()) {
logger.debug("Query travel time: {}ms, ES took: {}ms", swQuery.getElapsedTime(), response.getTook().getMillis()); logger.debug("Query travel time: {}ms, ES took: {}ms", swQuery.getElapsedTime(), response.getTook().getMillis());
} }
......
package eu.dariah.de.search.es.service; package eu.dariah.de.search.es.service;
import static org.springframework.util.Assert.*;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
...@@ -19,10 +21,9 @@ import org.elasticsearch.search.aggregations.AggregationBuilders; ...@@ -19,10 +21,9 @@ import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import de.unibamberg.minf.dme.model.base.Element; import de.unibamberg.minf.dme.model.base.Element;
import de.unibamberg.minf.dme.model.base.Nonterminal;
import de.unibamberg.minf.processing.model.base.Resource; import de.unibamberg.minf.processing.model.base.Resource;
import eu.dariah.de.search.Constants.AggregationTypes; import eu.dariah.de.search.Constants.AggregationTypes;
import eu.dariah.de.search.Constants.QueryTypes; import eu.dariah.de.search.Constants.QueryTypes;
...@@ -77,8 +78,8 @@ public class SearchServiceImpl implements SearchService { ...@@ -77,8 +78,8 @@ public class SearchServiceImpl implements SearchService {
@Override @Override
public List<QueryBuilder> generateQuery(QueryTypes[] queryTypes, String[] paths, String[] expressions, Operator[] operators, ExtendedDatamodelContainer dmc) { public List<QueryBuilder> generateQuery(QueryTypes[] queryTypes, String[] paths, String[] expressions, Operator[] operators, ExtendedDatamodelContainer dmc) {
Assert.isTrue(queryTypes.length == paths.length); isTrue(queryTypes.length == paths.length, () -> "Arrays for query types and paths must be equal in length");
Assert.isTrue(paths.length == expressions.length); isTrue(paths.length == expressions.length, () -> "Arrays for paths and expressions must be equal in length");
Element root = dmc.getOrRenderProcessingRoot(false); Element root = dmc.getOrRenderProcessingRoot(false);
...@@ -98,7 +99,7 @@ public class SearchServiceImpl implements SearchService { ...@@ -98,7 +99,7 @@ public class SearchServiceImpl implements SearchService {
@Override @Override
public AggregationBuilder generateAggregation(AggregationTypes aggregationType, String path, String aggregationName, int size, ExtendedDatamodelContainer dmc, List<AggregationBuilder> immediateAggs) { public AggregationBuilder generateAggregation(AggregationTypes aggregationType, String path, String aggregationName, int size, ExtendedDatamodelContainer dmc, List<AggregationBuilder> immediateAggs) {
List<AggregationBuilder> builders = this.generateAggregation(new AggregationTypes[] {aggregationType}, new String[] {path}, new String[] {aggregationName}, new int[] {size}, dmc, immediateAggs); List<AggregationBuilder> builders = this.generateAggregation(new AggregationTypes[] {aggregationType}, new String[] {path}, new String[] {aggregationName}, new int[] {size}, dmc, immediateAggs);
if (builders==null || builders.size()==0) { if (builders==null || builders.isEmpty()) {
return null; return null;
} }
return builders.get(0); return builders.get(0);
...@@ -106,8 +107,8 @@ public class SearchServiceImpl implements SearchService { ...@@ -106,8 +107,8 @@ public class SearchServiceImpl implements SearchService {
@Override @Override
public List<AggregationBuilder> generateAggregation(AggregationTypes[] aggregationTypes, String[] paths, String[] aggregationNames, int[] sizes, ExtendedDatamodelContainer dmc, List<AggregationBuilder> immediateAggs) { public List<AggregationBuilder> generateAggregation(AggregationTypes[] aggregationTypes, String[] paths, String[] aggregationNames, int[] sizes, ExtendedDatamodelContainer dmc, List<AggregationBuilder> immediateAggs) {
Assert.isTrue(aggregationTypes.length == paths.length); isTrue(aggregationTypes.length == paths.length, () -> "Arrays for aggregation types and paths must be equal in length");
Assert.isTrue(paths.length == aggregationNames.length); isTrue(paths.length == aggregationNames.length, () -> "Arrays for paths and aggregation names must be equal in length");
Element root = dmc.getOrRenderProcessingRoot(false); Element root = dmc.getOrRenderProcessingRoot(false);
...@@ -125,8 +126,8 @@ public class SearchServiceImpl implements SearchService { ...@@ -125,8 +126,8 @@ public class SearchServiceImpl implements SearchService {
} }
private List<QueryBuilder> createQueries(QueryTypes[] queryTypes, String[] paths, String[][] pathArrays, int depth, Object[] expressions, Operator[] operators, Element element) { private List<QueryBuilder> createQueries(QueryTypes[] queryTypes, String[] paths, String[][] pathArrays, int depth, Object[] expressions, Operator[] operators, Element element) {
Map<String, List<Integer>> keyIndicesMap = new HashMap<String, List<Integer>>(); Map<String, List<Integer>> keyIndicesMap = new HashMap<>();
List<QueryBuilder> queryList = new ArrayList<QueryBuilder>(); List<QueryBuilder> queryList = new ArrayList<>();
for (int i=0; i<pathArrays.length; i++) { for (int i=0; i<pathArrays.length; i++) {
try { try {
...@@ -175,12 +176,12 @@ public class SearchServiceImpl implements SearchService { ...@@ -175,12 +176,12 @@ public class SearchServiceImpl implements SearchService {
} }
if (ExtendedElement.class.isAssignableFrom(element.getClass()) && ((ExtendedElement)element).isNested() && queryList.size()>0) { if (ExtendedElement.class.isAssignableFrom(element.getClass()) && ((ExtendedElement)element).isNested() && queryList.size()>0) {
String nestedPath = ""; StringBuilder nestedPath = new StringBuilder();
for (int i=0; i<depth; i++) { for (int i=0; i<depth; i++) {
// Any path is ok as they are the same up to this depth // Any path is ok as they are the same up to this depth
nestedPath = nestedPath + pathArrays[0][i]; nestedPath.append(pathArrays[0][i]);
if (i<depth-1) { if (i<depth-1) {
nestedPath = nestedPath + "."; nestedPath.append(".");
} }
} }
...@@ -194,9 +195,9 @@ public class SearchServiceImpl implements SearchService { ...@@ -194,9 +195,9 @@ public class SearchServiceImpl implements SearchService {
cq = queryList.get(0); cq = queryList.get(0);
} }
NestedQueryBuilder nq = QueryBuilders.nestedQuery(nestedPath, cq, ScoreMode.Avg); NestedQueryBuilder nq = QueryBuilders.nestedQuery(nestedPath.toString(), cq, ScoreMode.Avg);
queryList = new ArrayList<QueryBuilder>(); queryList = new ArrayList<>();
queryList.add(nq); queryList.add(nq);
...@@ -213,13 +214,13 @@ public class SearchServiceImpl implements SearchService { ...@@ -213,13 +214,13 @@ public class SearchServiceImpl implements SearchService {
} else if (queryType.equals(QueryTypes.QueryStringQuery)) { } else if (queryType.equals(QueryTypes.QueryStringQuery)) {
return QueryBuilders.queryStringQuery(value.toString()).field(path).defaultOperator(operator); return QueryBuilders.queryStringQuery(value.toString()).field(path).defaultOperator(operator);
} else { } else {
throw new QueryExecutionException(String.format("Invalid QueryType: not implemented {}", queryType.toString())); throw new QueryExecutionException(String.format("Invalid QueryType: not implemented %s", queryType.toString()));
} }
} }
private List<AggregationBuilder> createAggregations(AggregationTypes[] aggregationTypes, String[] paths, String[][] pathArrays, int depth, String[] aggregationNames, int[] sizes, Element element, List<AggregationBuilder> immediateAggs) { private List<AggregationBuilder> createAggregations(AggregationTypes[] aggregationTypes, String[] paths, String[][] pathArrays, int depth, String[] aggregationNames, int[] sizes, Element element, List<AggregationBuilder> immediateAggs) {
Map<String, List<Integer>> keyIndicesMap = new HashMap<String, List<Integer>>(); Map<String, List<Integer>> keyIndicesMap = new HashMap<>();
List<AggregationBuilder> aggregationList = new ArrayList<AggregationBuilder>(); List<AggregationBuilder> aggregationList = new ArrayList<>();
for (int i=0; i<pathArrays.length; i++) { for (int i=0; i<pathArrays.length; i++) {
if (pathArrays[i]==null) { if (pathArrays[i]==null) {
...@@ -308,7 +309,7 @@ public class SearchServiceImpl implements SearchService { ...@@ -308,7 +309,7 @@ public class SearchServiceImpl implements SearchService {
} else if (aggregationType.equals(AggregationTypes.SignificantTermsAggregation)) { } else if (aggregationType.equals(AggregationTypes.SignificantTermsAggregation)) {
return AggregationBuilders.significantTerms(name).field(path).size(size); return AggregationBuilders.significantTerms(name).field(path).size(size);
} else { } else {
throw new QueryExecutionException(String.format("Invalid AggregationType: not implemented {}", aggregationType.toString())); throw new QueryExecutionException(String.format("Invalid AggregationType: not implemented %s", aggregationType.toString()));
} }
} }
} }
\ No newline at end of file
...@@ -32,7 +32,7 @@ log: ...@@ -32,7 +32,7 @@ log:
# pattern: "%-4relative [%thread] %-5level %logger{35}[%line]- %msg%n" # pattern: "%-4relative [%thread] %-5level %logger{35}[%line]- %msg%n"
indexing: indexing:
baseDir: /tmp/indexing_logs baseDir: /tmp/indexing_logs
#log_queries: false
indexing: indexing:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment