Commit 194279ea authored by Gradl, Tobias's avatar Gradl, Tobias
Browse files

Merge branch 'spring_boot' into 'make_modular'

410: Convert to Spring Boot library and application (OPENED)

See merge request !16
parents 17fde40e 2a6b61a8
Pipeline #23423 passed with stage
in 28 seconds
plugins {
//id 'io.spring.dependency-management' version "1.0.10.RELEASE"
//id 'org.springframework.boot' version "2.3.5.RELEASE" apply false
id "nebula.ospackage" version "8.4.1" apply false
id 'io.spring.dependency-management' version "1.0.10.RELEASE"
id 'org.springframework.boot' version "2.3.5.RELEASE" apply false
id "nebula.ospackage" version "8.4.1" apply false
}
allprojects {
apply plugin: 'eclipse'
......@@ -12,39 +13,38 @@ allprojects {
repositories {
mavenLocal()
maven {
maven {
url = 'https://minfba.de.dariah.eu/nexus/repository/minfba-central/'
}
}
ext {
coreVersion = "5.4.1-SNAPSHOT"
gtfVersion = "1.6.2-SNAPSHOT"
processingVersion = "4.0.2-SNAPSHOT"
colregModelVersion = "3.15.0-RELEASE"
dariahSpVersion = "1.4.0-SNAPSHOT"
springVersion = "4.3.6.RELEASE"
jacksonVersion = "2.9.6"
slf4jVersion = "1.7.22"
commonsLangVersion = "3.3.2"
httpComponentsVersion = "4.5.5"
elasticsearchVersion = "7.3.0"
logbackVersion = "1.1.3"
lombokVersion = "1.18.12"
mavenRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-central/'
releasesRepo = "https://minfba.de.dariah.eu/nexus/repository/minfba-releases/"
snapshotsRepo = "https://minfba.de.dariah.eu/nexus/repository/minfba-snapshots/"
aptReleasesRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-apt-releases/'
aptTestingRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-apt-testing/'
yumRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-yum/'
repoUser = project.hasProperty('nexususer') ? project.getProperty('nexususer') : '?';
repoPass = project.hasProperty('nexuspass') ? project.getProperty('nexuspass') : ''
coreVersion = "6.1-SNAPSHOT"
gtfVersion = "2.0.0-SNAPSHOT"
processingVersion = "4.1.0-SNAPSHOT"
colregModelVersion = "4.3.4-RELEASE"
dariahSpVersion = "2.1.6-RELEASE"
jsonAssertVersion = "1.5.0"
jodaTimeVersion = "2.10.10"
commonsTextVersion = "1.9"
commonsCompressVersion = "1.20"
commonsIoVersion = "2.8.0"
commonsCodecVersion = "1.15"
tikaVersion = "1.25"
mockitoVersion = "3.8.0"
mavenRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-central/'
releasesRepo = "https://minfba.de.dariah.eu/nexus/repository/minfba-releases/"
snapshotsRepo = "https://minfba.de.dariah.eu/nexus/repository/minfba-snapshots/"
aptReleasesRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-apt-releases/'
aptTestingRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-apt-testing/'
yumRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-yum/'
repoUser = project.hasProperty('nexususer') ? project.getProperty('nexususer') : '?';
repoPass = project.hasProperty('nexuspass') ? project.getProperty('nexuspass') : ''
// Filled dynamically by packaging tasks
debFile = ""
// Filled dynamically by packaging tasks
debFile = ""
rpmFile = ""
aptRepo = ""
}
......@@ -54,13 +54,45 @@ allprojects {
subprojects {
apply plugin: 'java'
apply plugin: 'maven-publish'
/*apply plugin: 'io.spring.dependency-management'
apply plugin: 'io.spring.dependency-management'
// https://stackoverflow.com/a/53824670
configurations {
implementation {
exclude group: 'xml-apis', module: 'xml-apis'
exclude group: 'xml-apis', module: 'xml-apis-ext'
exclude group: 'xerces', module: 'xercesImpl'
}
}
java {
sourceCompatibility = JavaVersion.VERSION_11
}
tasks.withType(JavaCompile) {
options.encoding = 'UTF-8'
}
dependencyManagement {
task sourcesJar(type: Jar, dependsOn: classes) {
classifier = 'sources'
from sourceSets.main.allSource
}
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier = 'javadoc'
from javadoc.destinationDir
}
artifacts {
archives sourcesJar
archives javadocJar
}
dependencyManagement {
imports {
mavenBom(org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES)
}
}*/
}
publishing {
publications {
......@@ -111,27 +143,4 @@ subprojects {
}
}
}
java {
sourceCompatibility = JavaVersion.VERSION_1_8
}
tasks.withType(JavaCompile) {
options.encoding = 'UTF-8'
}
task sourcesJar(type: Jar, dependsOn: classes) {
classifier = 'sources'
from sourceSets.main.allSource
}
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier = 'javadoc'
from javadoc.destinationDir
}
artifacts {
archives sourcesJar
archives javadocJar
}
}
......@@ -4,57 +4,87 @@ plugins {
description = "DARIAH-DE Generic Search - Core library"
/*configurations.all {
transitive = false
}*/
dependencies {
implementation "de.unibamberg.minf.core:core-metamodel:$coreVersion"
implementation "de.unibamberg.minf.core:core-web:$coreVersion"
implementation "de.unibamberg.minf.core:core-util:$coreVersion"
implementation "de.unibamberg.minf.gtf:gtf-base:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-core:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-extension-file:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-extension-wiki:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-extension-geo:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-extension-nlp:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-extension-vocabulary:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-extension-dai:$gtfVersion"
implementation "de.unibamberg.minf.processing:processing-core:$processingVersion"
implementation "de.unibamberg.minf.processing:processing-adapters:$processingVersion"
implementation "eu.dariah.de:colreg-model:$colregModelVersion"
implementation "eu.dariah.de:dariahsp-core:$dariahSpVersion"
implementation "org.springframework:spring-context:$springVersion"
implementation "org.springframework:spring-beans:$springVersion"
implementation "org.springframework:spring-core:$springVersion"
implementation "org.apache.httpcomponents:httpclient:$httpComponentsVersion"
implementation "org.elasticsearch:elasticsearch:$elasticsearchVersion"
implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:$elasticsearchVersion"
implementation "org.springframework.data:spring-data-mongodb:1.10.0.RELEASE"
implementation "org.slf4j:slf4j-api:$slf4jVersion"
implementation "joda-time:joda-time:2.9.9"
implementation "joda-time:joda-time-jsptags:1.1.1"
implementation "commons-io:commons-io:2.5"
implementation "org.apache.commons:commons-compress:1.15"
implementation "com.fasterxml.jackson.core:jackson-core:$jacksonVersion"
implementation "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion"
implementation "com.fasterxml.jackson.core:jackson-annotations:$jacksonVersion"
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:$jacksonVersion"
implementation "com.fasterxml.jackson.datatype:jackson-datatype-joda:$jacksonVersion"
implementation "org.skyscreamer:jsonassert:1.4.0"
implementation "org.apache.tika:tika-core:1.22"
implementation "org.apache.tika:tika-parsers:1.22"
testImplementation "org.springframework:spring-test:$springVersion"
testImplementation "junit:junit:4.12"
testImplementation "org.mockito:mockito-all:1.10.19"
testImplementation "org.elasticsearch.plugin:transport-netty4-client:$elasticsearchVersion"
testImplementation "org.slf4j:jcl-over-slf4j:$slf4jVersion"
testImplementation "org.slf4j:log4j-over-slf4j:$slf4jVersion"
testImplementation "ch.qos.logback:logback-core:$logbackVersion"
testImplementation "ch.qos.logback:logback-classic:$logbackVersion"
compileOnly "javax.servlet:servlet-api:2.5"
// MINF
api "de.unibamberg.minf.core:core-metamodel:$coreVersion"
api "de.unibamberg.minf.core:core-web:$coreVersion"
api "de.unibamberg.minf.core:core-util:$coreVersion"
api "eu.dariah.de:colreg-model:$colregModelVersion"
api "de.unibamberg.minf.processing:processing-core:$processingVersion"
api "de.unibamberg.minf.processing:processing-adapters:$processingVersion"
api "de.unibamberg.minf.gtf:gtf-base:$gtfVersion"
api "de.unibamberg.minf.gtf:gtf-core:$gtfVersion"
api "de.unibamberg.minf.gtf:gtf-extension-file:$gtfVersion"
api "de.unibamberg.minf.gtf:gtf-extension-wiki:$gtfVersion"
api "de.unibamberg.minf.gtf:gtf-extension-geo:$gtfVersion"
api "de.unibamberg.minf.gtf:gtf-extension-nlp:$gtfVersion"
api "de.unibamberg.minf.gtf:gtf-extension-vocabulary:$gtfVersion"
api "de.unibamberg.minf.gtf:gtf-extension-dai:$gtfVersion"
api "eu.dariah.de:dariahsp-core:$dariahSpVersion"
// Spring & data
api "org.springframework:spring-core"
api "org.springframework:spring-context"
api "org.springframework:spring-beans"
api "org.springframework:spring-web"
api "org.springframework:spring-webmvc"
api "org.springframework.data:spring-data-commons"
api "org.springframework.data:spring-data-mongodb"
api "org.springframework.data:spring-data-elasticsearch"
api "org.mongodb:mongodb-driver-sync"
api "org.mongodb:bson"
api "org.hibernate.validator:hibernate-validator"
api "javax.validation:validation-api"
api "org.apache.httpcomponents:httpclient"
// Jackson
api "com.fasterxml.jackson.core:jackson-core"
api "com.fasterxml.jackson.core:jackson-databind"
api "com.fasterxml.jackson.core:jackson-annotations"
// Apache Commons
api "org.apache.commons:commons-compress:$commonsCompressVersion"
api "org.apache.commons:commons-text:$commonsTextVersion"
api "commons-io:commons-io:$commonsIoVersion"
api "commons-codec:commons-codec:$commonsCodecVersion"
// Other
api "org.skyscreamer:jsonassert:$jsonAssertVersion"
api "org.slf4j:slf4j-api"
api "joda-time:joda-time:$jodaTimeVersion"
api "org.apache.tika:tika-core:$tikaVersion"
api "org.apache.tika:tika-parsers:$tikaVersion"
api "org.elasticsearch:elasticsearch"
compileOnly "javax.servlet:javax.servlet-api"
// Testing
testImplementation "org.springframework:spring-test"
testImplementation "org.junit.jupiter:junit-jupiter-api"
testImplementation "org.junit.jupiter:junit-jupiter-engine"
testImplementation "org.mockito:mockito-core:$mockitoVersion"
testImplementation "org.mockito:mockito-junit-jupiter:$mockitoVersion"
testRuntimeOnly "org.junit.platform:junit-platform-commons:1.7.0"
compileOnly "org.projectlombok:lombok:$lombokVersion"
annotationProcessor "org.projectlombok:lombok:$lombokVersion"
testCompileOnly "org.projectlombok:lombok:$lombokVersion"
// Lombok
compileOnly "org.projectlombok:lombok"
annotationProcessor "org.projectlombok:lombok"
testCompileOnly "org.projectlombok:lombok"
}
jar {
enabled = true
}
publishing {
publications {
maven(MavenPublication) {
from(components.java)
artifact(sourcesJar) {}
artifact(javadocJar) {}
}
}
}
\ No newline at end of file
......@@ -33,7 +33,9 @@ public class Constants {
public static final String PRESENTATION_FILTERS_KEY = RootElementKeys.PRESENTATION + ".Presentation.Filters";
public static final String ADMINISTRATOR = "ROLE_ADMINISTRATOR";
public static final String CONTRIBUTOR = "ROLE_CONTRIBUTOR";
public static final String USER = "ROLE_USER";
public static final String ES_ELEMENT_CONTAINER_KEY = "properties";
......@@ -78,7 +80,7 @@ public class Constants {
colregName = m;
}
public boolean equalsName(String otherName) {
return (otherName == null) ? false : colregName.equals(otherName);
return otherName != null && colregName.equals(otherName);
}
@Override
......@@ -91,14 +93,14 @@ public class Constants {
CONTENT ("_content"),
INTEGRATIONS ("_integrations"),
PRESENTATION ("_presentation"),
META ("_meta");
META (ELEMENT_KEY_META);
private final String key;
private RootElementKeys(String k) {
key = k;
}
public boolean equalsName(String otherName) {
return (otherName == null) ? false : key.equals(otherName);
return otherName != null && key.equals(otherName);
}
public String toString() {
......@@ -116,7 +118,7 @@ public class Constants {
this.elementName = elementName;
}
public boolean equalsName(String otherName) {
return (otherName == null) ? false : elementName.equals(otherName);
return otherName != null && elementName.equals(otherName);
}
public String toString() {
return this.elementName;
......
......@@ -5,12 +5,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import de.unibamberg.minf.dme.model.base.Grammar;
import de.unibamberg.minf.dme.model.grammar.GrammarContainer;
import de.unibamberg.minf.dme.model.serialization.MappingContainer;
import eu.dariah.de.search.api.client.base.BaseDmeClient;
import eu.dariah.de.search.model.ExtendedMappingContainer;
......@@ -22,28 +20,9 @@ public class DmeMappingClient extends BaseDmeClient<MappingContainer, MappingCon
private List<ExtendedMappingContainer> currentMappings;
private Map<String, Map<String, Grammar>> importMappingsGrammarsMap;
@Value("${url.dme}")
private String dmeUrl;
@Value("${api.dme.mapping_byST}")
private String fetchMappingBySourceAndTarget;
@Value("${api.dme.mapping_byT}")
private String fetchMappingByTarget;
@Value("${api.dme.mapping_byS}")
private String fetchMappingBySource;
@Value("${api.dme.mapping}")
private String fetchMappingDetails;
@Value("${api.dme.mappings}")
private String fetchAllUrl;
@Override protected String getFetchAllUrl() { return this.fetchAllUrl; }
@Override protected String getFetchDetailsUrl() { return this.fetchMappingDetails; }
@Override protected String getPingUrl() { return this.dmeUrl; }
@Override protected String getFetchAllUrl() { return this.apiConfig.getDme().getMappingsUrl(); }
@Override protected String getFetchDetailsUrl() { return this.apiConfig.getDme().getMappingUrl(); }
@Override protected String getPingUrl() { return this.apiConfig.getDme().getBaseUrl(); }
public List<ExtendedMappingContainer> getCurrentMappings() { return currentMappings; }
......
......@@ -5,7 +5,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
......@@ -13,7 +12,6 @@ import org.springframework.util.Assert;
import de.unibamberg.minf.dme.model.base.Element;
import de.unibamberg.minf.dme.model.base.Grammar;
import de.unibamberg.minf.dme.model.base.ModelElement;
import de.unibamberg.minf.dme.model.base.Nonterminal;
import de.unibamberg.minf.dme.model.datamodel.base.Datamodel;
import de.unibamberg.minf.dme.model.grammar.GrammarContainer;
import de.unibamberg.minf.dme.model.reference.Reference;
......@@ -29,28 +27,14 @@ public class DmeModelClient extends BaseDmeClient<DatamodelReferenceContainer, D
private List<ExtendedDatamodelContainer> currentDatamodels;
private Map<String, Map<String, GrammarContainer>> importDatamodelsGrammarsMap;
@Value("${url.dme}")
private String dmeUrl;
@Value("${api.dme.models}")
private String fetchAllUrl;
@Value("${api.dme.model}")
private String fetchModelDetails;
@Value("${api.dme.autocreate:true}")
private boolean autocreateModels;
public List<ExtendedDatamodelContainer> getCurrentDatamodels() { return currentDatamodels; }
public void setCurrentDatamodels(List<ExtendedDatamodelContainer> currentDatamodels) { this.currentDatamodels = currentDatamodels; }
@Override protected String getFetchAllUrl() { return this.fetchAllUrl; }
@Override protected String getFetchDetailsUrl() { return this.fetchModelDetails; }
@Override protected String getPingUrl() { return this.dmeUrl; }
@Override protected String getFetchAllUrl() { return this.apiConfig.getDme().getModelsUrl(); }
@Override protected String getFetchDetailsUrl() { return this.apiConfig.getDme().getModelUrl(); }
@Override protected String getPingUrl() { return this.apiConfig.getDme().getBaseUrl(); }
public DmeModelClient() {
super(DatamodelReferenceContainer.class, (new DatamodelReferenceContainer[0]).getClass());
......
......@@ -8,18 +8,21 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.client.RestTemplate;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.unibamberg.minf.core.util.Stopwatch;
import eu.dariah.de.search.automation.base.ListeningSyncService;
import eu.dariah.de.search.config.ApiConfigProperties;
import eu.dariah.de.search.pojo.ApiStatusPojo;
public abstract class BaseApiClientImpl<TBase, TExt> implements ApiClient {
protected final Logger logger = LoggerFactory.getLogger(this.getClass());
protected ObjectMapper apiObjectMapper;
@Autowired protected RestTemplate restTemplate;
@Autowired protected ObjectMapper objectMapper;
@Autowired protected ApiConfigProperties apiConfig;
private ListeningSyncService syncListener;
private Class<TExt> entityClass;
......@@ -30,9 +33,12 @@ public abstract class BaseApiClientImpl<TBase, TExt> implements ApiClient {
public void setSyncListener(ListeningSyncService baseSyncService) { this.syncListener = baseSyncService; }
public BaseApiClientImpl(Class<TExt> entityClass, Class<? extends TBase[]> arrayClass) {
protected BaseApiClientImpl(Class<TExt> entityClass, Class<? extends TBase[]> arrayClass) {
this.entityClass = entityClass;
this.arrayClass = arrayClass;
this.apiObjectMapper = new ObjectMapper()
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
}
protected void notifyListenerSyncFailed() {
......@@ -83,7 +89,7 @@ public abstract class BaseApiClientImpl<TBase, TExt> implements ApiClient {
// Workaround with explicit type conversion because RestTemplate sometimes uses superclass of entityClass
// TExt result = restTemplate.getForObject(String.format(this.getFetchDetailsUrl(), id), entityClass);
String result = restTemplate.getForObject(String.format(this.getFetchDetailsUrl(), id), String.class);
return objectMapper.readValue(result, entityClass);
return apiObjectMapper.readValue(result, entityClass);
} catch (Exception e) {
logger.error(String.format("Error while fetching details [%s] for id [%s]: %s", this.getFetchDetailsUrl(), id, e.getMessage()));
return null;
......
......@@ -96,6 +96,8 @@ public class CollectionSyncService extends BaseSyncService<Collection, Collectio
boolean saveCollection;
List<Endpoint> collectionEndpoints;
List<Dataset> endpointDatasets;
List<String> cmCollection;
Matcher m;
for (Collection c : collections) {
// Deletions can be executed right away
if (c.isDeleted()) {
......@@ -133,6 +135,14 @@ public class CollectionSyncService extends BaseSyncService<Collection, Collectio
e.getDatasets().remove(ds);
}
}
if (c.getImageUrl()!=null && ( c.isNew() || c.isUpdate() || ds.isNew() )) {
m = Constants.ONLINE_FILE_PATTERN.matcher(c.getImageUrl());
if (m.matches()) {
cmCollection = new ArrayList<>();
cmCollection.add(c.getImageUrl());
cachedImageService.cacheImage(c.getId(), e.getId(), ds.getId(), null, cmCollection);
}
}
}
}
}
......@@ -145,16 +155,6 @@ public class CollectionSyncService extends BaseSyncService<Collection, Collectio
if (saveCollection) {
this.collectionService.saveCollection(c);
if (c.getImageUrl()!=null) {
Matcher m = Constants.ONLINE_FILE_PATTERN.matcher(c.getImageUrl());
if (m.matches()) {
List<String> cmCollection = new ArrayList<>();
cmCollection.add(c.getImageUrl());
cachedImageService.cacheImage(c.getId(), null, null, null, cmCollection);
}
}
}
}
}
......
......@@ -101,12 +101,14 @@ public abstract class BaseSyncService<TModel extends Identifiable, TApi extends
@Override
public void destroy() throws Exception {
try {
syncExecutor.shutdown();
// Wait until all threads are finished
while (!syncExecutor.isTerminated()) {}
} catch (final Exception e) {
logger.error("Error closing sync executor", e);
if (syncExecutor!=null) {
try {
syncExecutor.shutdown();
// Wait until all threads are finished
while (!syncExecutor.isTerminated()) {}
} catch (final Exception e) {
logger.error("Error closing sync executor", e);
}
}
}
......
package eu.dariah.de.search.config;
import eu.dariah.de.search.config.nested.ColregConfigProperties;
import eu.dariah.de.search.config.nested.DmeConfigProperties;
import lombok.Data;
@Data
public class ApiConfigProperties {
private ColregConfigProperties colreg;
private DmeConfigProperties dme;
}
package eu.dariah.de.search.config;
import eu.dariah.de.search.config.nested.CrawlingAutomationConfigProperties;
import lombok.Data;
@Data
public class CrawlingConfigProperties {
private CrawlingAutomationConfigProperties automation;
private boolean debugging = false;
private int maxThreads = 4;
private int timeout = 172800; // 48 hours
}
package eu.dariah.de.search.config;
import eu.dariah.de.search.config.nested.IndexingLogConfigProperties;
import lombok.Data;
@Data
public class LogConfigProperties {
private IndexingLogConfigProperties indexing;
private String logFile = "search.log";
private String oldlogSuffix = "-%d{yyyy-MM-dd-HH-mm}-%i.log.gz";
private String dir;
private String pattern = "%d [%thread] %-5level %logger{36}[%line] - %msg%n";
private int maxHistory = 90;
private String totalSizeCap = "1GB";
private String maxFileSize = "10MB";
private boolean logQueries;
}
package eu.dariah.de.search.config;
import eu.dariah.de.search.config.nested.DatamodelConfigProperties;
import eu.dariah.de.search.config.nested.ImagesConfigProperties;
import eu.dariah.de.search.config.nested.IndexingConfigProperties;
import eu.dariah.de.search.config.nested.PathsConfigProperties;
import eu.dariah.de.search.config.nested.SearchConfigProperties;
import lombok.Data;
@Data
public class MainConfigProperties {
protected PathsConfigProperties paths;