Merge branch 'git' into 'master'

Git

See merge request !15

.gitlab-ci.yml

@@ -21,11 +21,15 @@ stages:
 build:
   stage: build
   script: ./gradlew assemble
+  only:
+    - master
 
 deploy:
   stage: deploy
   script:
     - ./gradlew publish -x test :search-docs:publish $NEXUS_CREDENTIALS
+  only:
+    - master
 
 docs:
   stage: deploy

build.gradle

@@ -18,7 +18,7 @@ plugins {
 
 allprojects {
     group = 'eu.dariah.de.minfba'
-    version = '3.9.6-SNAPSHOT'
+    version = '3.11.0-SNAPSHOT'
     description = 'DARIAH-DE Generic Search'
     
     ext {
@@ -43,7 +43,7 @@ repositories {
 sourceCompatibility = '1.8'
 
 ext {
-    searchCommonsVersion = "1.19.5-SNAPSHOT"
+    searchCommonsVersion = "1.20.0-SNAPSHOT"
 
     springVersion = "4.3.6.RELEASE"
         hibernateValidatorVersion = "5.2.2.Final"

i18n @ 7738d941

-Subproject commit 2d2667043e0dd783ddc352ae03310d32f82f1513
+Subproject commit 7738d94162679e2af797f954aacdb5926b59b943

src/main/resources/search_local_auth.yml

@@ -21,6 +21,7 @@ paths:
   crawls: ${paths.data}/crawls
   pictures: ${paths.data}/images
   downloads: ${paths.data}/downloads
+  backups: ${paths.data}/backups
   gtf:
     parse_errors: ${paths.data}/parseErrors
 

src/main/resources/search_saml_auth.yml

@@ -21,6 +21,7 @@ paths:
   crawls: ${paths.data}/crawls
   pictures: ${paths.data}/images
   downloads: ${paths.data}/downloads
+  backups: ${paths.data}/backups
   gtf:
     parse_errors: ${paths.data}/parseErrors
 

src/main/resources/search_static_custom.yml

@@ -21,6 +21,7 @@ paths:
   crawls: ${paths.data}/crawls
   pictures: ${paths.data}/images
   downloads: ${paths.data}/downloads
+  backups: ${paths.data}/backups
   gtf:
     parse_errors: ${paths.data}/parseErrors
 

src/main/resources/spring/processing-context.xml

@@ -63,6 +63,7 @@
 		<value>http://purl.org/cld/accpol/partial</value>
 	</util:list>
 
+
 	<bean id="timedCrawlManager" class="eu.dariah.de.search.crawling.TimedCrawlManagerImpl">
 		<property name="debugging" value="${crawling.debugging:false}" />
 		<property name="autocrawlOnline" value="${crawling.automation.online:true}" />
@@ -70,10 +71,10 @@
 		<property name="syncInterval" value="${crawling.automation.sync_interval:300}" />
 		<property name="timeout" value="${crawling.timeout:172800}" />
 		<property name="maxPoolSize" value="${crawling.max_threads:4}" />
-		<property name="offlineProcessingChains">
+		<!-- <property name="offlineProcessingChains">
 			<map>
 				<entry key="OAI-PMH" value="indexCleaner,xmlBatchFileProcessor" />
-				<!-- <entry key="XML" value="fileUnpacker,fileUnarchiver,xmlChunker,indexCleaner,xmlBatchFileProcessor" /> -->
+				<entry key="Git Repository" value="fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
 				<entry key="XML" value="fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
 				<entry key="JSON" value="fileUnpacker,fileUnarchiver,indexCleaner,jsonBatchFileProcessor" />		
 				<entry key="CSV" value="fileUnpacker,fileUnarchiver,indexCleaner,csvBatchFileProcessor" />
@@ -84,15 +85,35 @@
 		<property name="onlineProcessingChains">
 			<map>
 				<entry key="OAI-PMH" value="oaipmhCrawler,indexCleaner,xmlBatchFileProcessor" />
-				<!-- <entry key="XML" value="fileDownloader,fileUnpacker,fileUnarchiver,xmlChunker,indexCleaner,xmlBatchFileProcessor" /> -->
-				<entry key="XML" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
+				<entry key="Git Repository" value="gitCrawler, fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
+				<entry key="XML" value="fileDownloader,fileUnpacker,fileUnarchiver,xmlChunker,indexCleaner,xmlBatchFileProcessor" />
 				<entry key="JSON" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,jsonBatchFileProcessor" />	
 				<entry key="CSV" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,csvBatchFileProcessor" />
 				<entry key="TSV" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,tsvBatchFileProcessor" />
 				<entry key="TEXT" value="fileDownloader,fileUnpacker,fileUnarchiver,indexCleaner,textBatchFileProcessor" />
 			</map>
+		</property> -->
+		<property name="accessChains">
+			<map>
+				<entry key="OAI-PMH" value="oaipmhCrawler" />
+				<entry key="Git Repository" value="gitCrawler" />
+				<entry key="Online file" value="fileDownloader" />
+			</map>
+		</property>
+		<property name="fileProcessingChains">
+			<map>
+				<entry key="XML" value="fileUnpacker,fileUnarchiver,indexCleaner,xmlBatchFileProcessor" />
+				<entry key="JSON" value="fileUnpacker,fileUnarchiver,indexCleaner,jsonBatchFileProcessor" />		
+				<entry key="CSV" value="fileUnpacker,fileUnarchiver,indexCleaner,csvBatchFileProcessor" />
+				<entry key="TSV" value="fileUnpacker,fileUnarchiver,indexCleaner,tsvBatchFileProcessor" />
+				<entry key="TEXT" value="fileUnpacker,fileUnarchiver,indexCleaner,textBatchFileProcessor" />
+			</map>
 		</property>
 	</bean>
+	
+	<util:list id="antiPatterns" value-type="java.lang.String">
+		<value>.git/**</value>
+	</util:list>
 		
 	<bean id="indexCleaner" class="eu.dariah.de.search.crawling.crawler.IndexCleaner" scope="prototype" />
 	
@@ -100,6 +121,10 @@
 		<property name="politenessTimespan" value="${crawling.politeness_timespan:500}" />
 	</bean>
 	
+	<bean id="gitCrawler" class="eu.dariah.de.search.crawling.crawler.GitCrawlerImpl" scope="prototype" />
+	<bean id="gitRepositoryAdapter" class="de.unibamberg.minf.processing.git.adapter.GitRepositoryAdapterImpl" scope="prototype" />
+	
+	
 	<bean id="fileDownloader" class="eu.dariah.de.search.crawling.files.FileDownloader" scope="prototype" />
 	<bean id="fileUnpacker" class="eu.dariah.de.search.crawling.files.FileUnpacker" scope="prototype" />
 	<bean id="fileUnarchiver" class="eu.dariah.de.search.crawling.files.FileUnarchiver" scope="prototype" />

src/main/resources/spring/root-context.xml

@@ -38,6 +38,11 @@
 		<constructor-arg name="mongoConverter" ref="mongoMoxydomainConverter" />
 	</bean>
 
+	<bean class="eu.dariah.de.search.updates.UpdateServiceImpl">
+		<property name="backupsBasePath" value="${paths.backups:${paths.data}/backups}" />
+		<property name="database" value="${mongo.database}" />
+	</bean>
+	
 	
 	<bean id="messageSource" class="de.unibamberg.minf.core.web.localization.MessageSource">
 	    <property name="basenames" value="/themes/${theme:dariah-theme}/i18n/theme,classpath:i18n/messages" />

_search-common @ 8b07645b

-Subproject commit 4898eb68d5407e2f77ff84e2089cf7c1a5b057eb
+Subproject commit 8b07645b03f58758abb31cd710bdb5d294a4c1a4

resources @ c1d09b38

-Subproject commit 6a4a58f1cf0ed673ea0185db15371e03add7405f
+Subproject commit c1d09b387118415602ea98108e5b074ed7397bd4