Commit bea31cf3 authored by Gradl, Tobias's avatar Gradl, Tobias
Browse files

Elements as potential roots, not only Nonterminals

parent d2e37fac
Pipeline #15409 passed with stages
in 6 minutes and 22 seconds
...@@ -11,11 +11,12 @@ plugins { ...@@ -11,11 +11,12 @@ plugins {
id 'java' id 'java'
//id 'io.spring.dependency-management' version '1.0.8.RELEASE' //id 'io.spring.dependency-management' version '1.0.8.RELEASE'
id 'maven-publish' id 'maven-publish'
id 'eclipse'
} }
group = 'eu.dariah.de.minfba' group = 'eu.dariah.de.minfba'
version = '1.18.0-SNAPSHOT' version = '1.19.0-SNAPSHOT'
description = 'DARIAH-DE Search Commons' description = 'DARIAH-DE Search Commons'
sourceCompatibility = '1.8' sourceCompatibility = '1.8'
...@@ -27,9 +28,9 @@ repositories { ...@@ -27,9 +28,9 @@ repositories {
} }
ext { ext {
coreVersion = "5.3.0-SNAPSHOT" coreVersion = "5.4.1-SNAPSHOT"
gtfVersion = "1.6.0-SNAPSHOT" gtfVersion = "1.6.0-SNAPSHOT"
processingVersion = "3.8.2-SNAPSHOT" processingVersion = "3.9.2-SNAPSHOT"
colregModelVersion = "3.13.0-SNAPSHOT" colregModelVersion = "3.13.0-SNAPSHOT"
dariahSpVersion = "1.3.0-SNAPSHOT" dariahSpVersion = "1.3.0-SNAPSHOT"
......
rootProject.name = 'search-commons' rootProject.name = 'search-commons'
/*includeBuild('../core') {
dependencySubstitution {
substitute module('de.unibamberg.minf.core:core-metamodel') with project(':core-metamodel')
substitute module('de.unibamberg.minf.core:core-util') with project(':core-util')
substitute module('de.unibamberg.minf.core:core-web') with project(':core-web')
}
}
includeBuild('../processing') {
dependencySubstitution {
substitute module('de.unibamberg.minf.processing:processing') with project(':')
}
}*/
\ No newline at end of file
...@@ -196,7 +196,7 @@ public class CollectionEditorController extends BaseController { ...@@ -196,7 +196,7 @@ public class CollectionEditorController extends BaseController {
for (DatasetPojo ds : ep.getDatasetPojos()) { for (DatasetPojo ds : ep.getDatasetPojos()) {
esc = datamodelService.findById(ds.getId()); esc = datamodelService.findById(ds.getId());
ds.setDocs(datamodelService.getDocumentCount(esc.getIndexName(), epId)); ds.setDocs(datamodelService.getDocumentCount(esc.getIndexName(), ep.getId()));
if (dmId==null || ds.getId().equals(dmId)) { if (dmId==null || ds.getId().equals(dmId)) {
dmId = ds.getId(); dmId = ds.getId();
model.addAttribute("selectedDsId", ds.getId()); model.addAttribute("selectedDsId", ds.getId());
......
...@@ -93,10 +93,12 @@ public class ItemController extends BaseController { ...@@ -93,10 +93,12 @@ public class ItemController extends BaseController {
List<Resource> titles = ResourceHelper.findRecursive(res, "Title"); List<Resource> titles = ResourceHelper.findRecursive(res, "Title");
List<Resource> links = ResourceHelper.findRecursive(res, "Link"); List<Resource> links = ResourceHelper.findRecursive(res, "Link");
List<Resource> images = ResourceHelper.findRecursive(res, "Images.Resource"); List<Resource> images = ResourceHelper.findRecursive(res, "Images.Resource");
List<Resource> frames = ResourceHelper.findRecursive(res, "EmbeddedFrame");
model.addAttribute("titles", titles); model.addAttribute("titles", titles);
model.addAttribute("links", links); model.addAttribute("links", links);
model.addAttribute("images", images); model.addAttribute("images", images);
model.addAttribute("frames", frames);
if (titles!=null && !titles.isEmpty()) { if (titles!=null && !titles.isEmpty()) {
itemIdTitleMap.put(itemId, titles); itemIdTitleMap.put(itemId, titles);
......
...@@ -79,7 +79,7 @@ public class SearchServiceImpl extends BaseEsServiceImpl implements SearchServic ...@@ -79,7 +79,7 @@ public class SearchServiceImpl extends BaseEsServiceImpl implements SearchServic
Assert.isTrue(queryTypes.length == paths.length); Assert.isTrue(queryTypes.length == paths.length);
Assert.isTrue(paths.length == expressions.length); Assert.isTrue(paths.length == expressions.length);
Nonterminal root = dmc.getOrRenderProcessingRoot(false); Element root = dmc.getOrRenderProcessingRoot(false);
String[][] pathArrays = new String[paths.length][]; String[][] pathArrays = new String[paths.length][];
String[] pathArray; String[] pathArray;
...@@ -108,7 +108,7 @@ public class SearchServiceImpl extends BaseEsServiceImpl implements SearchServic ...@@ -108,7 +108,7 @@ public class SearchServiceImpl extends BaseEsServiceImpl implements SearchServic
Assert.isTrue(aggregationTypes.length == paths.length); Assert.isTrue(aggregationTypes.length == paths.length);
Assert.isTrue(paths.length == aggregationNames.length); Assert.isTrue(paths.length == aggregationNames.length);
Nonterminal root = dmc.getOrRenderProcessingRoot(false); Element root = dmc.getOrRenderProcessingRoot(false);
String[][] pathArrays = new String[paths.length][]; String[][] pathArrays = new String[paths.length][];
String[] pathArray; String[] pathArray;
......
...@@ -58,7 +58,7 @@ public class DatamodelPreparationServiceImpl implements DatamodelPreparationServ ...@@ -58,7 +58,7 @@ public class DatamodelPreparationServiceImpl implements DatamodelPreparationServ
public void extendDatamodel(ExtendedDatamodelContainer dmc, List<Resource> resources) { public void extendDatamodel(ExtendedDatamodelContainer dmc, List<Resource> resources) {
Assert.notNull(resources); Assert.notNull(resources);
Nonterminal root = this.findRoot(dmc); Element root = this.findRoot(dmc);
Reference rRoot = ReferenceHelper.findRootElementReference(dmc.getRoot()); Reference rRoot = ReferenceHelper.findRootElementReference(dmc.getRoot());
Map<String, ModelElement> reusedElementMap = new HashMap<>(); Map<String, ModelElement> reusedElementMap = new HashMap<>();
...@@ -70,7 +70,7 @@ public class DatamodelPreparationServiceImpl implements DatamodelPreparationServ ...@@ -70,7 +70,7 @@ public class DatamodelPreparationServiceImpl implements DatamodelPreparationServ
public void pruneDatamodel(ExtendedDatamodelContainer dmc, List<Resource> resources) { public void pruneDatamodel(ExtendedDatamodelContainer dmc, List<Resource> resources) {
Assert.notNull(resources); Assert.notNull(resources);
Nonterminal root = this.findRoot(dmc); Element root = this.findRoot(dmc);
this.pruneElementTree(root, resources); this.pruneElementTree(root, resources);
} }
...@@ -92,8 +92,8 @@ public class DatamodelPreparationServiceImpl implements DatamodelPreparationServ ...@@ -92,8 +92,8 @@ public class DatamodelPreparationServiceImpl implements DatamodelPreparationServ
datamodelService.saveOrUpdate(saveDmc); datamodelService.saveOrUpdate(saveDmc);
} }
private Nonterminal findRoot(ExtendedDatamodelContainer dmc) { private Element findRoot(ExtendedDatamodelContainer dmc) {
Nonterminal root; Element root;
if (dmc.isIncludeHeaders()) { if (dmc.isIncludeHeaders()) {
root = (Nonterminal)dmc.getOrRenderElementHierarchy(true); root = (Nonterminal)dmc.getOrRenderElementHierarchy(true);
} else { } else {
......
...@@ -150,7 +150,7 @@ public class MappingGenerationServiceImpl implements MappingGenerationService { ...@@ -150,7 +150,7 @@ public class MappingGenerationServiceImpl implements MappingGenerationService {
mcc = prepareIndexMetamodelMapping(dmc); mcc = prepareIndexMetamodelMapping(dmc);
} }
ObjectNode node = objMapper.createObjectNode(); ObjectNode node = objMapper.createObjectNode();
Nonterminal root; Element root;
if (dmc.isIncludeHeaders()) { if (dmc.isIncludeHeaders()) {
root = (Nonterminal)dmc.getOrRenderElementHierarchy(true); root = (Nonterminal)dmc.getOrRenderElementHierarchy(true);
} else { } else {
......
...@@ -8,6 +8,7 @@ import org.springframework.data.annotation.Transient; ...@@ -8,6 +8,7 @@ import org.springframework.data.annotation.Transient;
import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import de.unibamberg.minf.dme.model.base.BaseElement;
import de.unibamberg.minf.dme.model.base.Element; import de.unibamberg.minf.dme.model.base.Element;
import de.unibamberg.minf.dme.model.base.Identifiable; import de.unibamberg.minf.dme.model.base.Identifiable;
import de.unibamberg.minf.dme.model.base.ModelElement; import de.unibamberg.minf.dme.model.base.ModelElement;
...@@ -27,7 +28,7 @@ public class ExtendedDatamodelContainer extends DatamodelReferenceContainer impl ...@@ -27,7 +28,7 @@ public class ExtendedDatamodelContainer extends DatamodelReferenceContainer impl
private Map<String, List<String>> analyzerFieldMap; private Map<String, List<String>> analyzerFieldMap;
private Nonterminal processingRootElement, processingRootElementNoReuses; private Element processingRootElement, processingRootElementNoReuses;
@Override public String getId() { return this.getModel().getId(); } @Override public String getId() { return this.getModel().getId(); }
@Override public void setId(String id) { } @Override public void setId(String id) { }
...@@ -69,7 +70,7 @@ public class ExtendedDatamodelContainer extends DatamodelReferenceContainer impl ...@@ -69,7 +70,7 @@ public class ExtendedDatamodelContainer extends DatamodelReferenceContainer impl
@JsonIgnore @JsonIgnore
@Transient @Transient
public boolean isIncludeHeaders() { public boolean isIncludeHeaders() {
Nonterminal processingRoot = this.getOrRenderProcessingRoot(false); Element processingRoot = this.getOrRenderProcessingRoot(false);
if (processingRoot!=null) { if (processingRoot!=null) {
return processingRoot.isIncludeHeader(); return processingRoot.isIncludeHeader();
} }
...@@ -78,23 +79,23 @@ public class ExtendedDatamodelContainer extends DatamodelReferenceContainer impl ...@@ -78,23 +79,23 @@ public class ExtendedDatamodelContainer extends DatamodelReferenceContainer impl
@JsonIgnore @JsonIgnore
@Transient @Transient
public Nonterminal getOrRenderProcessingRoot() { public Element getOrRenderProcessingRoot() {
return this.getOrRenderProcessingRoot(false); return this.getOrRenderProcessingRoot(false);
} }
@JsonIgnore @JsonIgnore
@Transient @Transient
public Nonterminal getOrRenderProcessingRoot(boolean skippedReuses) { public Element getOrRenderProcessingRoot(boolean skippedReuses) {
if (this.getProcessingRootElement(skippedReuses)==null) { if (this.getProcessingRootElement(skippedReuses)==null) {
Element root = this.getOrRenderElementHierarchy(skippedReuses); Element root = this.getOrRenderElementHierarchy(skippedReuses);
if (root==null || !Nonterminal.class.isAssignableFrom(root.getClass())) { if (root==null || !Nonterminal.class.isAssignableFrom(root.getClass())) {
return null; return null;
} else { } else {
Nonterminal nRoot = (Nonterminal)root; Element nRoot = root;
if (nRoot.isProcessingRoot()) { if (nRoot.isProcessingRoot()) {
this.setProcessingRootElement(nRoot, skippedReuses); this.setProcessingRootElement(nRoot, skippedReuses);
} else { } else {
this.setProcessingRootElement(NonterminalImpl.findProcessingRoot(nRoot), skippedReuses); this.setProcessingRootElement(BaseElement.findProcessingRoot(nRoot), skippedReuses);
} }
return this.getProcessingRootElement(skippedReuses); return this.getProcessingRootElement(skippedReuses);
} }
...@@ -104,11 +105,11 @@ public class ExtendedDatamodelContainer extends DatamodelReferenceContainer impl ...@@ -104,11 +105,11 @@ public class ExtendedDatamodelContainer extends DatamodelReferenceContainer impl
} }
protected Nonterminal getProcessingRootElement(boolean skippedReuses) { protected Element getProcessingRootElement(boolean skippedReuses) {
return skippedReuses ? processingRootElementNoReuses : processingRootElement; return skippedReuses ? processingRootElementNoReuses : processingRootElement;
} }
protected void setProcessingRootElement(Nonterminal processingRoot, boolean skippedReuses) { protected void setProcessingRootElement(Element processingRoot, boolean skippedReuses) {
if (skippedReuses) { if (skippedReuses) {
this.processingRootElementNoReuses = processingRoot; this.processingRootElementNoReuses = processingRoot;
} else { } else {
......
...@@ -21,8 +21,9 @@ public class CollectingConsumptionServiceImpl implements MappedResourceConsumpti ...@@ -21,8 +21,9 @@ public class CollectingConsumptionServiceImpl implements MappedResourceConsumpti
} }
@Override @Override
public void consume(Resource obj) { public boolean consume(Resource obj) {
// TODO: unused, throw Exception? // TODO: unused, throw Exception?
return false;
} }
@Override @Override
...@@ -31,13 +32,14 @@ public class CollectingConsumptionServiceImpl implements MappedResourceConsumpti ...@@ -31,13 +32,14 @@ public class CollectingConsumptionServiceImpl implements MappedResourceConsumpti
} }
@Override @Override
public void consume(Resource rSource, Resource rTarget) { public boolean consume(Resource rSource, Resource rTarget) {
try { try {
mappedResourceMap.put(rSource.hashCode(), rTarget); mappedResourceMap.put(rSource.hashCode(), rTarget);
return true;
} catch (Exception e) { } catch (Exception e) {
System.err.println("something"); System.err.println("something");
} }
return false;
} }
} }
...@@ -7,7 +7,6 @@ import org.slf4j.LoggerFactory; ...@@ -7,7 +7,6 @@ import org.slf4j.LoggerFactory;
import de.unibamberg.minf.processing.consumption.MappedResourceConsumptionService; import de.unibamberg.minf.processing.consumption.MappedResourceConsumptionService;
import de.unibamberg.minf.processing.model.base.Resource; import de.unibamberg.minf.processing.model.base.Resource;
import eu.dariah.de.search.Constants;
import eu.dariah.de.search.Constants.RootElementKeys; import eu.dariah.de.search.Constants.RootElementKeys;
import eu.dariah.de.search.query.SearchFacetImpl; import eu.dariah.de.search.query.SearchFacetImpl;
...@@ -26,12 +25,12 @@ public class FacetResourceConsumptionService implements MappedResourceConsumptio ...@@ -26,12 +25,12 @@ public class FacetResourceConsumptionService implements MappedResourceConsumptio
} }
@Override @Override
public void consume(Resource rTarget) { public boolean consume(Resource rTarget) {
this.consume(null, rTarget); return this.consume(null, rTarget);
} }
@Override @Override
public void consume(Resource rSource, Resource rTarget) { public boolean consume(Resource rSource, Resource rTarget) {
try { try {
SearchFacetImpl sourceFacet = null; SearchFacetImpl sourceFacet = null;
if (rSource!=null && SearchFacetImpl.class.isAssignableFrom(rSource.getClass())) { if (rSource!=null && SearchFacetImpl.class.isAssignableFrom(rSource.getClass())) {
...@@ -41,9 +40,11 @@ public class FacetResourceConsumptionService implements MappedResourceConsumptio ...@@ -41,9 +40,11 @@ public class FacetResourceConsumptionService implements MappedResourceConsumptio
if (fcts!=null) { if (fcts!=null) {
this.consumedFacets.addAll(fcts); this.consumedFacets.addAll(fcts);
} }
return true;
} catch (Exception e) { } catch (Exception e) {
logger.error("Failed to consume search facet from resource", e); logger.error("Failed to consume search facet from resource", e);
} }
return false;
} }
@Override @Override
......
...@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired; ...@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import de.unibamberg.minf.dme.model.base.BaseElement;
import de.unibamberg.minf.dme.model.base.Element; import de.unibamberg.minf.dme.model.base.Element;
import de.unibamberg.minf.dme.model.base.Nonterminal; import de.unibamberg.minf.dme.model.base.Nonterminal;
import de.unibamberg.minf.dme.model.datamodel.NonterminalImpl; import de.unibamberg.minf.dme.model.datamodel.NonterminalImpl;
...@@ -132,7 +133,7 @@ public class FacetServiceImpl implements FacetService { ...@@ -132,7 +133,7 @@ public class FacetServiceImpl implements FacetService {
if (extS!=null && extS.getRoot()!=null) { if (extS!=null && extS.getRoot()!=null) {
Reference rRoot = ReferenceHelper.findRootElementReference(extS.getRoot()); Reference rRoot = ReferenceHelper.findRootElementReference(extS.getRoot());
// fillElement with skipAllReuses creates placeholder elements instead of dereferencing the potentially recursive trees // fillElement with skipAllReuses creates placeholder elements instead of dereferencing the potentially recursive trees
Nonterminal processingRoot = NonterminalImpl.findProcessingRoot((Nonterminal)ReferenceHelper.fillElement(rRoot, extS.getElements(), true)); Element processingRoot = BaseElement.findProcessingRoot((Nonterminal)ReferenceHelper.fillElement(rRoot, extS.getElements(), true));
this.generateFacet(facets, processingRoot, "", null); this.generateFacet(facets, processingRoot, "", null);
} }
...@@ -146,7 +147,7 @@ public class FacetServiceImpl implements FacetService { ...@@ -146,7 +147,7 @@ public class FacetServiceImpl implements FacetService {
// fillElement with skipAllReuses creates placeholder elements instead of dereferencing the potentially recursive trees // fillElement with skipAllReuses creates placeholder elements instead of dereferencing the potentially recursive trees
Nonterminal root = extS.getOrRenderProcessingRoot(true); Element root = extS.getOrRenderProcessingRoot(true);
return this.findFieldName(root, schemaElementId, ""); return this.findFieldName(root, schemaElementId, "");
......
...@@ -73,13 +73,14 @@ public class ResourceIndexingServiceImpl implements ResourceIndexingService { ...@@ -73,13 +73,14 @@ public class ResourceIndexingServiceImpl implements ResourceIndexingService {
} }
@Override @Override
public void consume(Resource resource) { public boolean consume(Resource resource) {
this.enrichmentService.consume(resource); this.enrichmentService.consume(resource);
// Chunk-processing incoming data // Chunk-processing incoming data
if (this.enrichmentService.getResourceBatch().size()>=BULK_CHUNK_SIZE) { if (this.enrichmentService.getResourceBatch().size()>=BULK_CHUNK_SIZE) {
this.commit(); this.commit();
} }
return true;
} }
@Override @Override
......
...@@ -15,11 +15,12 @@ public class CollectingResourceConsumptionService implements ResourceConsumption ...@@ -15,11 +15,12 @@ public class CollectingResourceConsumptionService implements ResourceConsumption
@Override @Override
public void consume(Resource res) { public boolean consume(Resource res) {
if (resources==null) { if (resources==null) {
resources = new ArrayList<Resource>(); resources = new ArrayList<Resource>();
} }
resources.add(res); resources.add(res);
return true;
} }
@Override @Override
......
...@@ -23,8 +23,9 @@ public class MappedResourceCollectionServiceImpl implements MappedResourceConsum ...@@ -23,8 +23,9 @@ public class MappedResourceCollectionServiceImpl implements MappedResourceConsum
} }
@Override @Override
public void consume(Resource obj) { public boolean consume(Resource obj) {
this.getResourcePairs().add(new MappedResourcePair(obj, null)); this.getResourcePairs().add(new MappedResourcePair(obj, null));
return true;
} }
@Override @Override
...@@ -33,8 +34,9 @@ public class MappedResourceCollectionServiceImpl implements MappedResourceConsum ...@@ -33,8 +34,9 @@ public class MappedResourceCollectionServiceImpl implements MappedResourceConsum
} }
@Override @Override
public void consume(Resource rSource, Resource rTarget) { public boolean consume(Resource rSource, Resource rTarget) {
this.getResourcePairs().add(new MappedResourcePair(rSource, rTarget)); this.getResourcePairs().add(new MappedResourcePair(rSource, rTarget));
return true;
} }
} }
...@@ -11,6 +11,7 @@ import org.springframework.beans.factory.annotation.Value; ...@@ -11,6 +11,7 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope; import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import de.unibamberg.minf.dme.model.base.Element;
import de.unibamberg.minf.dme.model.base.Nonterminal; import de.unibamberg.minf.dme.model.base.Nonterminal;
import de.unibamberg.minf.dme.model.datamodel.NonterminalImpl; import de.unibamberg.minf.dme.model.datamodel.NonterminalImpl;
import de.unibamberg.minf.processing.model.base.Resource; import de.unibamberg.minf.processing.model.base.Resource;
...@@ -140,24 +141,24 @@ public class ResourceEnrichmentServiceImpl implements ResourceEnrichmentService ...@@ -140,24 +141,24 @@ public class ResourceEnrichmentServiceImpl implements ResourceEnrichmentService
} }
@Override @Override
public void consume(Resource resource) { public boolean consume(Resource resource) {
this.consume(resource, null); return this.consume(resource, null);
} }
@Override @Override
public void consume(Resource resource, Resource rTransformed) { public boolean consume(Resource resource, Resource rTransformed) {
if (resource==null) { if (resource==null) {
return; return false;
} }
if (this.isCurrentStage(ResourceEnrichmentStages.RESOURCE_COLLECTION)) { if (this.isCurrentStage(ResourceEnrichmentStages.RESOURCE_COLLECTION)) {
ResourceContainer rc = this.collectResource(resource); ResourceContainer rc = this.collectResource(resource);
rc.setCurrentStage(this.getCurrentStage()); rc.setCurrentStage(this.getCurrentStage());
resourceBatch.add(rc); resourceBatch.add(rc);
return; return true;
} }
if (rTransformed==null) { if (rTransformed==null) {
return; return false;
} }
ResourceContainer rc = (ResourceContainer)resource; ResourceContainer rc = (ResourceContainer)resource;
...@@ -171,7 +172,7 @@ public class ResourceEnrichmentServiceImpl implements ResourceEnrichmentService ...@@ -171,7 +172,7 @@ public class ResourceEnrichmentServiceImpl implements ResourceEnrichmentService
if (!resourceBatch.contains(rc)) { if (!resourceBatch.contains(rc)) {
resourceBatch.add(rc); resourceBatch.add(rc);
} }
return true;
} }
@Override @Override
...@@ -189,7 +190,7 @@ public class ResourceEnrichmentServiceImpl implements ResourceEnrichmentService ...@@ -189,7 +190,7 @@ public class ResourceEnrichmentServiceImpl implements ResourceEnrichmentService
if (