Skip to content

GitLab

Commit 9565156b authored by Gradl, Tobias's avatar Gradl, Tobias
Browse files

410: Convert to Spring Boot library and application (OPENED) 

Task-Url: #410
parent 17fde40e
Pipeline #22556 passed with stage
in 49 seconds
Hide whitespace changes
Inline Side-by-side
build.gradle
View file @ 9565156b
plugins {
//id 'io.spring.dependency-management' version "1.0.10.RELEASE"
//id 'org.springframework.boot' version "2.3.5.RELEASE" apply false
id "nebula.ospackage" version "8.4.1" apply false
id 'io.spring.dependency-management' version "1.0.10.RELEASE"
id 'org.springframework.boot' version "2.3.5.RELEASE" apply false
id "nebula.ospackage" version "8.4.1" apply false
}
allprojects {
apply plugin: 'eclipse'
......@@ -12,39 +13,29 @@ allprojects {
repositories {
mavenLocal()
maven {
maven {
url = 'https://minfba.de.dariah.eu/nexus/repository/minfba-central/'
}
}
ext {
coreVersion = "5.4.1-SNAPSHOT"
gtfVersion = "1.6.2-SNAPSHOT"
processingVersion = "4.0.2-SNAPSHOT"
colregModelVersion = "3.15.0-RELEASE"
dariahSpVersion = "1.4.0-SNAPSHOT"
springVersion = "4.3.6.RELEASE"
jacksonVersion = "2.9.6"
slf4jVersion = "1.7.22"
commonsLangVersion = "3.3.2"
httpComponentsVersion = "4.5.5"
elasticsearchVersion = "7.3.0"
logbackVersion = "1.1.3"
lombokVersion = "1.18.12"
coreVersion = "6.1-SNAPSHOT"
gtfVersion = "2.0.0-SNAPSHOT"
processingVersion = "4.1.0-SNAPSHOT"
colregModelVersion = "4.2-SNAPSHOT"
dariahSpVersion = "2.1.4-SNAPSHOT"
mavenRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-central/'
releasesRepo = "https://minfba.de.dariah.eu/nexus/repository/minfba-releases/"
snapshotsRepo = "https://minfba.de.dariah.eu/nexus/repository/minfba-snapshots/"
aptReleasesRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-apt-releases/'
aptTestingRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-apt-testing/'
yumRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-yum/'
repoUser = project.hasProperty('nexususer') ? project.getProperty('nexususer') : '?';
repoPass = project.hasProperty('nexuspass') ? project.getProperty('nexuspass') : ''
mavenRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-central/'
releasesRepo = "https://minfba.de.dariah.eu/nexus/repository/minfba-releases/"
snapshotsRepo = "https://minfba.de.dariah.eu/nexus/repository/minfba-snapshots/"
aptReleasesRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-apt-releases/'
aptTestingRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-apt-testing/'
yumRepo = 'https://minfba.de.dariah.eu/nexus/repository/minfba-yum/'
repoUser = project.hasProperty('nexususer') ? project.getProperty('nexususer') : '?';
repoPass = project.hasProperty('nexuspass') ? project.getProperty('nexuspass') : ''
// Filled dynamically by packaging tasks
debFile = ""
// Filled dynamically by packaging tasks
debFile = ""
rpmFile = ""
aptRepo = ""
}
......@@ -54,13 +45,36 @@ allprojects {
subprojects {
apply plugin: 'java'
apply plugin: 'maven-publish'
/*apply plugin: 'io.spring.dependency-management'
apply plugin: 'io.spring.dependency-management'
java {
sourceCompatibility = JavaVersion.VERSION_11
}
tasks.withType(JavaCompile) {
options.encoding = 'UTF-8'
}
task sourcesJar(type: Jar, dependsOn: classes) {
classifier = 'sources'
from sourceSets.main.allSource
}
dependencyManagement {
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier = 'javadoc'
from javadoc.destinationDir
}
artifacts {
archives sourcesJar
archives javadocJar
}
dependencyManagement {
imports {
mavenBom(org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES)
}
}*/
}
publishing {
publications {
......@@ -111,27 +125,4 @@ subprojects {
}
}
}
java {
sourceCompatibility = JavaVersion.VERSION_1_8
}
tasks.withType(JavaCompile) {
options.encoding = 'UTF-8'
}
task sourcesJar(type: Jar, dependsOn: classes) {
classifier = 'sources'
from sourceSets.main.allSource
}
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier = 'javadoc'
from javadoc.destinationDir
}
artifacts {
archives sourcesJar
archives javadocJar
}
}
search-core/build.gradle
View file @ 9565156b
......@@ -4,10 +4,65 @@ plugins {
description = "DARIAH-DE Generic Search - Core library"
// https://stackoverflow.com/a/53824670
configurations {
implementation {
exclude group: 'xml-apis', module: 'xml-apis'
exclude group: 'xml-apis', module: 'xml-apis-ext'
exclude group: 'xerces', module: 'xercesImpl'
}
}
/*configurations.all {
transitive = false
}*/
ext {
jsonAssertVersion = "1.5.0"
jodaTimeVersion = "2.10.10"
commonsTextVersion = "1.9"
commonsCompressVersion = "1.20"
commonsIoVersion = "2.8.0"
commonsCodecVersion = "1.15"
tikaVersion = "1.25"
mockitoVersion = "3.8.0"
}
dependencies {
implementation "de.unibamberg.minf.core:core-metamodel:$coreVersion"
implementation "de.unibamberg.minf.core:core-metamodel:$coreVersion"
implementation "de.unibamberg.minf.core:core-web:$coreVersion"
implementation "de.unibamberg.minf.core:core-util:$coreVersion"
implementation "eu.dariah.de:colreg-model:$colregModelVersion"
implementation "org.springframework:spring-core"
implementation "org.springframework:spring-context"
implementation "org.springframework:spring-beans"
implementation "org.springframework:spring-web"
implementation "org.springframework:spring-webmvc"
implementation "org.springframework.data:spring-data-commons"
implementation "org.springframework.data:spring-data-mongodb"
implementation "org.springframework.data:spring-data-elasticsearch"
implementation "org.mongodb:mongodb-driver-sync"
implementation "org.mongodb:bson"
api "com.fasterxml.jackson.core:jackson-core"
api "com.fasterxml.jackson.core:jackson-databind"
api "com.fasterxml.jackson.core:jackson-annotations"
implementation "org.apache.commons:commons-compress:$commonsCompressVersion"
implementation "org.apache.commons:commons-text:$commonsTextVersion"
implementation "commons-io:commons-io:$commonsIoVersion"
implementation "commons-codec:commons-codec:$commonsCodecVersion"
implementation "org.skyscreamer:jsonassert:$jsonAssertVersion"
implementation "org.hibernate.validator:hibernate-validator"
implementation "javax.validation:validation-api"
implementation "de.unibamberg.minf.gtf:gtf-base:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-core:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-extension-file:$gtfVersion"
......@@ -16,45 +71,72 @@ dependencies {
implementation "de.unibamberg.minf.gtf:gtf-extension-nlp:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-extension-vocabulary:$gtfVersion"
implementation "de.unibamberg.minf.gtf:gtf-extension-dai:$gtfVersion"
implementation "de.unibamberg.minf.processing:processing-core:$processingVersion"
implementation "de.unibamberg.minf.processing:processing-adapters:$processingVersion"
implementation "eu.dariah.de:colreg-model:$colregModelVersion"
implementation "de.unibamberg.minf.processing:processing-adapters:$processingVersion"
implementation "eu.dariah.de:dariahsp-core:$dariahSpVersion"
implementation "org.springframework:spring-context:$springVersion"
implementation "org.springframework:spring-beans:$springVersion"
implementation "org.springframework:spring-core:$springVersion"
implementation "org.apache.httpcomponents:httpclient:$httpComponentsVersion"
implementation "org.elasticsearch:elasticsearch:$elasticsearchVersion"
implementation "org.elasticsearch.client:elasticsearch-rest-high-level-client:$elasticsearchVersion"
implementation "org.springframework.data:spring-data-mongodb:1.10.0.RELEASE"
implementation "org.slf4j:slf4j-api:$slf4jVersion"
implementation "joda-time:joda-time:2.9.9"
implementation "joda-time:joda-time-jsptags:1.1.1"
implementation "commons-io:commons-io:2.5"
implementation "org.apache.commons:commons-compress:1.15"
implementation "com.fasterxml.jackson.core:jackson-core:$jacksonVersion"
implementation "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion"
implementation "com.fasterxml.jackson.core:jackson-annotations:$jacksonVersion"
implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:$jacksonVersion"
implementation "com.fasterxml.jackson.datatype:jackson-datatype-joda:$jacksonVersion"
implementation "org.skyscreamer:jsonassert:1.4.0"
implementation "org.apache.tika:tika-core:1.22"
implementation "org.apache.tika:tika-parsers:1.22"
testImplementation "org.springframework:spring-test:$springVersion"
testImplementation "junit:junit:4.12"
testImplementation "org.mockito:mockito-all:1.10.19"
testImplementation "org.elasticsearch.plugin:transport-netty4-client:$elasticsearchVersion"
testImplementation "org.slf4j:jcl-over-slf4j:$slf4jVersion"
testImplementation "org.slf4j:log4j-over-slf4j:$slf4jVersion"
testImplementation "ch.qos.logback:logback-core:$logbackVersion"
testImplementation "ch.qos.logback:logback-classic:$logbackVersion"
compileOnly "javax.servlet:servlet-api:2.5"
compileOnly "org.projectlombok:lombok:$lombokVersion"
annotationProcessor "org.projectlombok:lombok:$lombokVersion"
testCompileOnly "org.projectlombok:lombok:$lombokVersion"
implementation "org.apache.httpcomponents:httpclient"
//implementation "org.elasticsearch:elasticsearch"
implementation "org.slf4j:slf4j-api"
implementation "joda-time:joda-time:$jodaTimeVersion"
//implementation "joda-time:joda-time-jsptags:1.1.1"
//implementation "commons-io:commons-io"
//implementation "org.apache.commons:commons-compress"
//implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-xml"
//implementation "com.fasterxml.jackson.datatype:jackson-datatype-joda"
//implementation "org.skyscreamer:jsonassert:1.4.0"
implementation "org.apache.tika:tika-core:$tikaVersion"
implementation "org.apache.tika:tika-parsers:$tikaVersion"
implementation "org.elasticsearch:elasticsearch"
//testImplementation "org.springframework.boot:spring-boot-starter-test"
testImplementation "org.springframework:spring-test"
testImplementation "org.junit.jupiter:junit-jupiter-api"
testImplementation "org.junit.jupiter:junit-jupiter-engine"
testImplementation "org.mockito:mockito-core:$mockitoVersion"
testImplementation "org.mockito:mockito-junit-jupiter:$mockitoVersion"
//testImplementation "junit:junit:4.12"
//
//testImplementation "org.junit.jupiter:junit-jupiter-engine"
///testImplementation "org.junit.jupiter:junit-jupiter-api"
//testImplementation "org.junit.jupiter:junit-jupiter-params"
//testImplementation "org.slf4j:jcl-over-slf4j"
//testImplementation "org.slf4j:log4j-over-slf4j"
//testImplementation "ch.qos.logback:logback-core"
//testImplementation "ch.qos.logback:logback-classic"
compileOnly "javax.servlet:javax.servlet-api"
compileOnly "org.projectlombok:lombok"
annotationProcessor "org.projectlombok:lombok"
testCompileOnly "org.projectlombok:lombok"
}
jar {
enabled = true
}
publishing {
publications {
maven(MavenPublication) {
from(components.java)
artifact(sourcesJar) {}
artifact(javadocJar) {}
}
}
}
\ No newline at end of file
search-core/src/main/java/eu/dariah/de/search/api/client/CollectionSyncClient.java
View file @ 9565156b
package eu.dariah.de.search.api.client;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.joda.time.DateTime;
import org.joda.time.Period;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.type.TypeReference;
import de.unibamberg.minf.core.web.localization.LocaleConverter;
import eu.dariah.de.colreg.pojo.AccessPojo;
import eu.dariah.de.colreg.pojo.AccrualPojo;
import eu.dariah.de.colreg.pojo.DatamodelPojo;
import eu.dariah.de.colreg.pojo.api.CollectionApiPojo;
import eu.dariah.de.colreg.pojo.api.ExtendedCollectionApiPojo;
import eu.dariah.de.colreg.pojo.api.results.CollectionApiResultPojo;
import eu.dariah.de.search.Constants.AccessMethods;
import eu.dariah.de.search.api.client.base.BaseApiClientImpl;
import eu.dariah.de.search.model.Collection;
import eu.dariah.de.search.model.Dataset;
import eu.dariah.de.search.model.Endpoint;
@Component
@Scope(value="prototype")
public class CollectionSyncClient extends BaseApiClientImpl<CollectionApiPojo, ExtendedCollectionApiPojo> {
private static final String defaultUnclosedFrequencyKey = "_defaultUnclosed";
private Map<String, Period> updateFrequencyMap;
private List<String> knownUpdatePolicies;
private List<Collection> currentCollections;
@Value("${url.colreg}")
private String colregUrl;
@Value("${datamodels.special.oai_dc:#{null}}")
private String oaidcModel;
@Value("${api.colreg.fetch_all}")
private String fetchAllUrl;
@Value("${api.colreg.fetch_detail}")
private String fetchDetailUrl;
public List<Collection> getCurrentCollections() { return currentCollections; }
public void setCurrentCollections(List<Collection> currentCollections) { this.currentCollections = currentCollections; }
public Map<String, Period> getUpdateFrequencyMap() { return updateFrequencyMap; }
public void setUpdateFrequencyMap(Map<String, Period> updateFrequencyMap) { this.updateFrequencyMap = updateFrequencyMap; }
public List<String> getKnownUpdatePolicies() { return knownUpdatePolicies; }
public void setKnownUpdatePolicies(List<String> knownUpdatePolicies) { this.knownUpdatePolicies = knownUpdatePolicies; }
@Override protected String getFetchAllUrl() { return fetchAllUrl; }
@Override protected String getFetchDetailsUrl() { return fetchDetailUrl; }
@Override protected String getPingUrl() { return this.colregUrl; }
public CollectionSyncClient() {
super(ExtendedCollectionApiPojo.class, (new CollectionApiPojo[0]).getClass());
}
protected ExtendedCollectionApiPojo fetchDetails(String id) {
if (id==null || this.getFetchDetailsUrl()==null) {
return null;
}
try {
// Workaround with explicit type conversion because RestTemplate sometimes uses superclass of entityClass
// TExt result = restTemplate.getForObject(String.format(this.getFetchDetailsUrl(), id), entityClass);
String result = restTemplate.getForObject(String.format(this.getFetchDetailsUrl(), id), String.class);
CollectionApiResultPojo<ExtendedCollectionApiPojo> rPojo = objectMapper.readValue(result, new TypeReference<CollectionApiResultPojo<ExtendedCollectionApiPojo>>() {});
if (rPojo.getContent()!=null && rPojo.getContent().size()>0) {
return rPojo.getContent().iterator().next();
}
return null;
} catch (Exception e) {
logger.error(String.format("Error while fetching details [%s] for id [%s]: %s", this.getFetchDetailsUrl(), id, e.getMessage()));
return null;
}
}
@Override
public CollectionApiPojo[] fetch(String requestUrl) {
try {
String result = restTemplate.getForObject(requestUrl, String.class);
CollectionApiResultPojo<CollectionApiPojo> rPojo = objectMapper.readValue(result, new TypeReference<CollectionApiResultPojo<CollectionApiPojo>>() {});
return rPojo.getContent().toArray(new CollectionApiPojo[0]);
} catch (Exception e) {
logger.error(String.format("Error while executing [%s]: %s", requestUrl, e.getMessage()));
return null;
}
}
@Override
protected void syncEntities(CollectionApiPojo[] fetchedCollections) {
int mergeCount = 0;
int outdateCount = 0;
int newCount = 0;
List<Collection> outdatedCollections = new ArrayList<>();
if (this.getCurrentCollections()!=null) {
outdatedCollections.addAll(this.getCurrentCollections());
}
if (fetchedCollections!=null) {
for (CollectionApiPojo fetchedCollection : fetchedCollections) {
boolean mergeOrUnchanged = false;
if (this.getCurrentCollections()!=null) {
for (Collection currentCollection : this.getCurrentCollections()) {
// Entity id match -> collection already imported
if (currentCollection.getColregEntityId().equals(fetchedCollection.getId())) {
mergeOrUnchanged = true;
outdatedCollections.remove(currentCollection);
// Entity id !match -> new version of collection exists, update required
//if (!currentCollection.getColregVersionId().equals(fetchedCollection.getVersionId())) {
this.mergeCollection(currentCollection, fetchedCollection.getId());
mergeCount++;
//}
break;
}
}
}
// New collections
if (!mergeOrUnchanged) {
this.importCollection(fetchedCollection.getId());
newCount++;
}
}
}
// Flag deleted collections
for (Collection currentCollection : outdatedCollections) {
currentCollection.setDeleted(true);
}
logger.info(String.format("Synchronization thread finished: %s new, %s merged, %s deleted", newCount, mergeCount, outdateCount));
}
private void importCollection(String foreignEntityId) {
Collection importedCollection = this.fetchAndConvertCollection(foreignEntityId);
importedCollection.setNew(true);
// Import only collections with accessible endpoints
if (importedCollection.getEndpoints()!=null && importedCollection.getEndpoints().size()>0) {
for (Endpoint endpoint : importedCollection.getEndpoints()) {
endpoint.setNew(true);
if (endpoint.getDatasets()!=null) {
// All endpoints and model references are newly imported -> crawls required
for (Dataset dataset : endpoint.getDatasets()) {
dataset.setNew(true);
}
}
}
this.getCurrentCollections().add(importedCollection);
}
}
private void mergeCollection(Collection cCurrent, String foreignEntityId) {
Collection cImported = this.fetchAndConvertCollection(foreignEntityId);
// Overwrite only fields that really come from CR, not those that are altered in GS
cCurrent.setColregVersionId(cImported.getColregVersionId());
cCurrent.setImageUrl(cImported.getImageUrl());
cCurrent.setNames(cImported.getNames());
cCurrent.setModified(cImported.getModified());
cCurrent.setUpdatePeriod(cImported.getUpdatePeriod());
cCurrent.setUpdate(true);
this.mergeEndpoints(cCurrent, cImported);
// Mark collections without endpoint as deleted
if (cCurrent.getEndpoints()==null || cCurrent.getEndpoints().size()==0) {
cCurrent.setDeleted(true);
}
}
private void mergeEndpoints(Collection cCurrent, Collection cFetched) {
List<Endpoint> deleteEndpoints = new ArrayList<>();
if (cCurrent.getEndpoints()!=null) {
deleteEndpoints.addAll(cCurrent.getEndpoints());
}
for (Endpoint eFetched : cFetched.getEndpoints()) {
boolean mergeOrUnchanged = false;
// Find updates
if (cCurrent.getEndpoints()!=null) {
for (Endpoint eCurrent : cCurrent.getEndpoints()) {
if (this.endpointsAreSame(eCurrent, eFetched)) {
mergeOrUnchanged = true;
this.mergeDatamodelReferences(eCurrent, eFetched);
deleteEndpoints.remove(eCurrent);
break;
}
}
}
if (!mergeOrUnchanged) {
// New one
if (cCurrent.getEndpoints()==null) {
cCurrent.setEndpoints(new ArrayList<>());
}
cCurrent.getEndpoints().add(eFetched);
eFetched.setNew(true);
}
}
// Flag deleted collections
if (!deleteEndpoints.isEmpty()) {
for (Endpoint deleteEndpoint : deleteEndpoints) {
deleteEndpoint.setDeleted(true);
}
}
}
private void mergeDatamodelReferences(Endpoint eCurrent, Endpoint eFetched) {
List<String> removeDatamodels = new ArrayList<>();
if (eCurrent.getDatasets()!=null) {
for (Dataset dataset : eCurrent.getDatasets()) {
removeDatamodels.add(dataset.getId());
}
}
if (eFetched.getDatasets()!=null) {
for (Dataset dataset : eFetched.getDatasets()) {
// Already pointing to datamodel
boolean contained = false;
for (Dataset dsCurrent : eCurrent.getDatasets()) {
if (this.datasetsAreSame(dsCurrent, dataset)) {
contained = true;
dsCurrent.setRemoteAlias(dataset.getRemoteAlias());
}
}
if (contained) {
removeDatamodels.remove(dataset.getId());
} else {
// New reference
dataset.setNew(true);
if (eCurrent.getDatasets()==null) {
eCurrent.setDatasets(new ArrayList<Dataset>());
}
eCurrent.getDatasets().add(dataset);
}
}
}
// Deleted reference to datamodel
if (removeDatamodels.size()>0) {
for (Dataset dataset : eCurrent.getDatasets()) {
if (removeDatamodels.contains(dataset.getId())) {
dataset.setDeleted(true);
}
}
}
}
private Collection fetchAndConvertCollection(String foreignEntityId) {
if (foreignEntityId==null) {
return null;
}
ExtendedCollectionApiPojo fetchedCollection = this.fetchDetails(foreignEntityId);
Collection convertedCollection = new Collection();
if (fetchedCollection.getTitles()!=null) {
convertedCollection.setNames(new HashMap<String, String>());
String languageCode;
for (String iso3Code : fetchedCollection.getTitles().keySet()) {
languageCode = LocaleConverter.getLanguageForIso3Code(iso3Code);
// Only keep translations that matter to the GS interface
if (languageCode!=null) {
convertedCollection.getNames().put(languageCode, fetchedCollection.getTitles().get(iso3Code));
} else {
convertedCollection.getNames().put(iso3Code, fetchedCollection.getTitles().get(iso3Code));
}