Commit 8c03b621 authored by Gradl, Tobias's avatar Gradl, Tobias

Merge branch 'git' into 'master'

Git

See merge request !15
parents e4f4ea3f 754dd05b
Pipeline #16235 passed with stages
in 7 minutes and 50 seconds
......@@ -21,11 +21,15 @@ stages:
build:
stage: build
script: ./gradlew assemble
only:
- master
deploy:
stage: deploy
script:
- ./gradlew publish -x test :search-docs:publish $NEXUS_CREDENTIALS
only:
- master
docs:
stage: deploy
......
......@@ -18,7 +18,7 @@ plugins {
allprojects {
group = 'eu.dariah.de.minfba'
version = '3.9.6-SNAPSHOT'
version = '3.11.0-SNAPSHOT'
description = 'DARIAH-DE Generic Search'
ext {
......@@ -43,7 +43,7 @@ repositories {
sourceCompatibility = '1.8'
ext {
searchCommonsVersion = "1.19.5-SNAPSHOT"
searchCommonsVersion = "1.20.0-SNAPSHOT"
springVersion = "4.3.6.RELEASE"
hibernateValidatorVersion = "5.2.2.Final"
......
Subproject commit 2d2667043e0dd783ddc352ae03310d32f82f1513
Subproject commit 7738d94162679e2af797f954aacdb5926b59b943
......@@ -21,6 +21,7 @@ paths:
crawls: ${paths.data}/crawls
pictures: ${paths.data}/images
downloads: ${paths.data}/downloads
backups: ${paths.data}/backups
gtf:
parse_errors: ${paths.data}/parseErrors
......
......@@ -21,6 +21,7 @@ paths:
crawls: ${paths.data}/crawls
pictures: ${paths.data}/images
downloads: ${paths.data}/downloads
backups: ${paths.data}/backups
gtf:
parse_errors: ${paths.data}/parseErrors
......
......@@ -21,6 +21,7 @@ paths:
crawls: ${paths.data}/crawls
pictures: ${paths.data}/images
downloads: ${paths.data}/downloads
backups: ${paths.data}/backups
gtf:
parse_errors: ${paths.data}/parseErrors
......
......@@ -63,6 +63,7 @@
<value>http://purl.org/cld/accpol/partial</value>
</util:list>
<bean id="timedCrawlManager" class="eu.dariah.de.search.crawling.TimedCrawlManagerImpl">
<property name="debugging" value="${crawling.debugging:false}" />
<property name="autocrawlOnline" value="${crawling.automation.online:true}" />
......@@ -70,10 +71,10 @@
<property name="syncInterval" value="${crawling.automation.sync_interval:300}" />
<property name="timeout" value="${crawling.timeout:172800}" />
<property name="maxPoolSize" value="${crawling.max_threads:4}" />
<property name="offlineProcessingChains">
<!-- <property name="offlineProcessingChains">
<map>
<entry key="OAI-PMH" value="indexCleaner,xmlBatchFileProcessor" />
<!-- <entry key="XML" value="fileUnpacker,fileUnarchiver,xmlChunker,indexCleaner,xmlBatchFileProcessor" /> -->
<entry key="Git Repository" value="fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
<entry key="XML" value="fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
<entry key="JSON" value="fileUnpacker,fileUnarchiver,indexCleaner,jsonBatchFileProcessor" />
<entry key="CSV" value="fileUnpacker,fileUnarchiver,indexCleaner,csvBatchFileProcessor" />
......@@ -84,15 +85,35 @@
<property name="onlineProcessingChains">
<map>
<entry key="OAI-PMH" value="oaipmhCrawler,indexCleaner,xmlBatchFileProcessor" />
<!-- <entry key="XML" value="fileDownloader,fileUnpacker,fileUnarchiver,xmlChunker,indexCleaner,xmlBatchFileProcessor" /> -->
<entry key="XML" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
<entry key="Git Repository" value="gitCrawler, fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
<entry key="XML" value="fileDownloader,fileUnpacker,fileUnarchiver,xmlChunker,indexCleaner,xmlBatchFileProcessor" />
<entry key="JSON" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,jsonBatchFileProcessor" />
<entry key="CSV" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,csvBatchFileProcessor" />
<entry key="TSV" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,tsvBatchFileProcessor" />
<entry key="TEXT" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,textBatchFileProcessor" />
</map>
</property> -->
<property name="accessChains">
<map>
<entry key="OAI-PMH" value="oaipmhCrawler" />
<entry key="Git Repository" value="gitCrawler" />
<entry key="Online file" value="fileDownloader" />
</map>
</property>
<property name="fileProcessingChains">
<map>
<entry key="XML" value="fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
<entry key="JSON" value="fileUnpacker,fileUnarchiver,indexCleaner,jsonBatchFileProcessor" />
<entry key="CSV" value="fileUnpacker,fileUnarchiver,indexCleaner,csvBatchFileProcessor" />
<entry key="TSV" value="fileUnpacker,fileUnarchiver,indexCleaner,tsvBatchFileProcessor" />
<entry key="TEXT" value="fileUnpacker,fileUnarchiver,indexCleaner,textBatchFileProcessor" />
</map>
</property>
</bean>
<util:list id="antiPatterns" value-type="java.lang.String">
<value>.git/**</value>
</util:list>
<bean id="indexCleaner" class="eu.dariah.de.search.crawling.crawler.IndexCleaner" scope="prototype" />
......@@ -100,6 +121,10 @@
<property name="politenessTimespan" value="${crawling.politeness_timespan:500}" />
</bean>
<bean id="gitCrawler" class="eu.dariah.de.search.crawling.crawler.GitCrawlerImpl" scope="prototype" />
<bean id="gitRepositoryAdapter" class="de.unibamberg.minf.processing.git.adapter.GitRepositoryAdapterImpl" scope="prototype" />
<bean id="fileDownloader" class="eu.dariah.de.search.crawling.files.FileDownloader" scope="prototype" />
<bean id="fileUnpacker" class="eu.dariah.de.search.crawling.files.FileUnpacker" scope="prototype" />
<bean id="fileUnarchiver" class="eu.dariah.de.search.crawling.files.FileUnarchiver" scope="prototype" />
......
......@@ -38,6 +38,11 @@
<constructor-arg name="mongoConverter" ref="mongoMoxydomainConverter" />
</bean>
<bean class="eu.dariah.de.search.updates.UpdateServiceImpl">
<property name="backupsBasePath" value="${paths.backups:${paths.data}/backups}" />
<property name="database" value="${mongo.database}" />
</bean>
<bean id="messageSource" class="de.unibamberg.minf.core.web.localization.MessageSource">
<property name="basenames" value="/themes/${theme:dariah-theme}/i18n/theme,classpath:i18n/messages" />
......
Subproject commit 4898eb68d5407e2f77ff84e2089cf7c1a5b057eb
Subproject commit 8b07645b03f58758abb31cd710bdb5d294a4c1a4
Subproject commit 6a4a58f1cf0ed673ea0185db15371e03add7405f
Subproject commit c1d09b387118415602ea98108e5b074ed7397bd4
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment