Commit 2dfbcb90 authored by Gradl, Tobias's avatar Gradl, Tobias
Browse files

419: Adapt to new generic access parameter concept of the CR (OPENED)

Task-Url: #419
parent 51315918
Pipeline #23317 passed with stage
in 30 seconds
......@@ -21,7 +21,7 @@ allprojects {
coreVersion = "6.1-SNAPSHOT"
gtfVersion = "2.0.0-SNAPSHOT"
processingVersion = "4.1.0-SNAPSHOT"
colregModelVersion = "4.2-SNAPSHOT"
colregModelVersion = "4.3.2-RELEASE"
dariahSpVersion = "2.1.4-SNAPSHOT"
jsonAssertVersion = "1.5.0"
......
......@@ -14,6 +14,7 @@ import org.springframework.stereotype.Component;
import com.fasterxml.jackson.core.type.TypeReference;
import de.unibamberg.minf.core.web.localization.LocaleConverter;
import eu.dariah.de.colreg.pojo.AccessParamPojo;
import eu.dariah.de.colreg.pojo.AccessPojo;
import eu.dariah.de.colreg.pojo.AccrualPojo;
import eu.dariah.de.colreg.pojo.DatamodelPojo;
......@@ -26,6 +27,7 @@ import eu.dariah.de.search.config.MainConfigProperties;
import eu.dariah.de.search.model.Collection;
import eu.dariah.de.search.model.Dataset;
import eu.dariah.de.search.model.Endpoint;
import eu.dariah.de.search.model.EndpointParam;
@Component
@Scope(value="prototype")
......@@ -333,11 +335,21 @@ public class CollectionSyncClient extends BaseApiClientImpl<CollectionApiPojo, E
}
}
e.setSet(accessPojo.getSet());
if (accessPojo.getParams()!=null) {
e.setParams(new ArrayList<>(accessPojo.getParams().size()));
EndpointParam param;
for (AccessParamPojo paramPojo : accessPojo.getParams()) {
param = new EndpointParam();
param.setParam(paramPojo.getKey());
param.setValue(paramPojo.getValue());
e.getParams().add(param);
}
}
e.setUrl(accessPojo.getUri());
e.setPatterns(accessPojo.getPatterns());
e.setDatasets(new ArrayList<Dataset>());
e.setDatasets(new ArrayList<>());
if (accessPojo.getDatamodels()!=null) {
for (DatamodelPojo m : accessPojo.getDatamodels()) {
e.getDatasets().add(new Dataset(m.getId(), m.getAlias()));
......@@ -376,9 +388,8 @@ public class CollectionSyncClient extends BaseApiClientImpl<CollectionApiPojo, E
private boolean endpointsAreSame(Endpoint ep1, Endpoint ep2) {
try {
return ep1.getAccessType().equals(ep2.getAccessType()) &&
ep1.getFileType().equals(ep2.getFileType()) &&
ep1.getUrl().equals(ep2.getUrl()) &&
(ep1.getSet()==null && ep2.getSet()==null || ep1.getSet().equals(ep2.getSet()) );
ep1.getFileType().equals(ep2.getFileType()) &&
ep1.renderAccessUrl().equals(ep2.renderAccessUrl());
} catch (Exception e) {
return false;
}
......
......@@ -191,21 +191,20 @@ public class CollectionEditorController extends BaseController {
List<Crawl> crawls = crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Online, CrawlCompleteFlag.Complete, CrawlErrorFlag.NoError, 1);
if (!crawls.isEmpty()) {
lastOnlineCrawl = crawls.get(0);
List<String> clearCrawlIds = new ArrayList<>();
crawls = crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Offline, CrawlCompleteFlag.Both, CrawlErrorFlag.Both, 0);
for (Crawl c : crawls) {
if (c.getModified().isBefore(lastOnlineCrawl.getModified())) {
clearCrawlIds.add(c.getId());
}
}
this.addCrawlId(clearCrawlIds, crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Both, CrawlCompleteFlag.Incomplete, CrawlErrorFlag.Both, 0));
this.addCrawlId(clearCrawlIds, crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Both, CrawlCompleteFlag.Both, CrawlErrorFlag.Error, 0));
for (String clearCrawlId : clearCrawlIds) {
crawlService.removeCrawlById(clearCrawlId);
}
}
List<String> clearCrawlIds = new ArrayList<>();
crawls = crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Offline, CrawlCompleteFlag.Both, CrawlErrorFlag.Both, 0);
for (Crawl c : crawls) {
if (lastOnlineCrawl==null || c.getModified().isBefore(lastOnlineCrawl.getModified())) {
clearCrawlIds.add(c.getId());
}
}
this.addCrawlId(clearCrawlIds, crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Both, CrawlCompleteFlag.Incomplete, CrawlErrorFlag.Both, 0));
this.addCrawlId(clearCrawlIds, crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Both, CrawlCompleteFlag.Both, CrawlErrorFlag.Error, 0));
for (String clearCrawlId : clearCrawlIds) {
crawlService.removeCrawlById(clearCrawlId);
}
ModelActionPojo result = new ModelActionPojo();
result.setSuccess(true);
......@@ -235,12 +234,14 @@ public class CollectionEditorController extends BaseController {
if (dmId==null || ds.getId().equals(dmId)) {
dmId = ds.getId();
model.addAttribute("selectedDsId", ds.getId());
model.addAttribute("ds", ds);
break;
}
}
if (epId==null || ep.getId().equals(epId)) {
epId = ep.getId();
model.addAttribute("selectedEpId", ep.getId());
model.addAttribute("ep", ep);
break;
}
}
......
......@@ -232,7 +232,7 @@ public class CrawlManagerImpl implements CrawlManager, ApplicationContextAware,
}
// Online but no access type detected
if (access==null && c.getBaseCrawlId()==null) {
logger.error(String.format("Unknown access tyüe [%s]; cancelling crawl", ep.getAccessType()));
logger.error(String.format("Unknown access type [%s]; cancelling crawl", ep.getAccessType()));
this.updateCrawl(c.getId(), ProcessingServiceStates.ERROR);
return null;
}
......
package eu.dariah.de.search.crawling.crawler;
import java.io.File;
import java.io.IOException;
import org.apache.commons.io.FileUtils;
import org.slf4j.MDC;
......@@ -57,8 +56,8 @@ public class GitCrawlerImpl extends GitRepositoryProcessingService implements Cr
@Override
public void init(Endpoint endpoint, Crawl crawl, ExtendedDatamodelContainer sc) {
this.setUrl(endpoint.getUrl());
this.setBranch(endpoint.getSet());
this.setUrl(endpoint.renderAccessUrl());
this.setBranch(endpoint.getSingleParamValue("branch"));
this.crawlId = crawl.getId();
this.setCrawlDir(crawlService.getCrawlDirPath(crawl));
......
package eu.dariah.de.search.crawling.crawler;
import java.net.URISyntaxException;
import org.slf4j.MDC;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -45,8 +47,8 @@ public class OaiPmhCrawlerImpl extends OaiPmhHarvestingService implements Crawle
@Override
public void init(Endpoint endpoint, Crawl crawl, ExtendedDatamodelContainer sc) {
this.setUrl(endpoint.getUrl());
this.setSet(endpoint.getSet());
this.setUrl(endpoint.renderAccessUrl());
this.setSet(endpoint.getSingleParamValue("set"));
this.crawlId = crawl.getId();
if (crawl.getPrefix()==null || crawl.getPrefix().trim().isEmpty()) {
......@@ -56,7 +58,7 @@ public class OaiPmhCrawlerImpl extends OaiPmhHarvestingService implements Crawle
this.initialized = false;
return;
} else {
logger.warn(String.format("Metadata prefix for OAI-PMH endpoint [%s] automatically detected [%s]", endpoint.getUrl(), prefix));
logger.warn(String.format("Metadata prefix for OAI-PMH endpoint [%s] automatically detected [%s]", this.getUrl(), prefix));
crawl.setPrefix(prefix);
crawlService.save(crawl);
}
......@@ -82,7 +84,7 @@ public class OaiPmhCrawlerImpl extends OaiPmhHarvestingService implements Crawle
}
String prefix = null;
OaiPmhResponseContainer oaiFormatsResponse = oaiPmhClient.listMetadataFormats(ep.getUrl(), null);
OaiPmhResponseContainer oaiFormatsResponse = oaiPmhClient.listMetadataFormats(ep.renderAccessUrl(), null);
if (oaiFormatsResponse!=null && oaiFormatsResponse.getFormats()!=null) {
for (OaiPmhMetadataFormat format : oaiFormatsResponse.getFormats()) {
if (format.getMetadataNamespace().trim().toLowerCase().equals(rootNs)) {
......
......@@ -70,7 +70,7 @@ public class FileDownloader extends BaseFileStreamCrawler implements Crawler {
try {
this.setupPaths(crawl);
this.inputURI = new URL(endpoint.getUrl()).toURI();
this.inputURI = new URL(endpoint.renderAccessUrl()).toURI();
this.initialized = true;
} catch (MalformedURLException | URISyntaxException e) {
......
package eu.dariah.de.search.model;
import java.net.URISyntaxException;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.http.client.utils.URIBuilder;
import org.springframework.data.annotation.Transient;
import de.unibamberg.minf.dme.model.base.Identifiable;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Data
public class Endpoint implements Identifiable {
private static final long serialVersionUID = 4977814286153993508L;
......@@ -15,12 +22,13 @@ public class Endpoint implements Identifiable {
private List<String> patterns;
private List<EndpointParam> params;
private String url;
private String accessType;
private String fileType;
private String set;
private String dateTimeFormatPattern;
private String updatePeriod;
......@@ -30,44 +38,47 @@ public class Endpoint implements Identifiable {
private boolean error;
private boolean unaccessible;
@Transient private boolean deleted;
public String renderAccessUrl() {
if (this.getUrl()==null) {
return null;
}
try {
URIBuilder b = new URIBuilder(this.getUrl());
if (params!=null) {
for (EndpointParam p : params) {
b.addParameter(p.getParam(), p.getValue());
}
}
return b.build().toString();
} catch (URISyntaxException e) {
log.error("Failed to build URL", e);
}
return null;
}
public List<EndpointParam> getParams(String param) {
return params==null ? null : this.params.stream()
.filter(p -> p.getParam().equals(param))
.collect(Collectors.toList());
}
@Override public String getId() { return id; }
@Override public void setId(String id) { this.id = id; }
public boolean isError() { return error; }
public void setError(boolean error) { this.error = error; }
public boolean isUnaccessible() { return unaccessible; }
public void setUnaccessible(boolean unaccessible) { this.unaccessible = unaccessible; }
public List<Dataset> getDatasets() { return datasets; }
public void setDatasets(List<Dataset> datasets) { this.datasets = datasets; }
public String getUrl() { return url; }
public void setUrl(String url) { this.url = url; }
public String getAccessType() { return accessType; }
public void setAccessType(String accessType) { this.accessType = accessType; }
public String getFileType() { return fileType; }
public void setFileType(String fileType) { this.fileType = fileType; }
public String getSet() { return set; }
public void setSet(String set) { this.set = set; }
public String getDateTimeFormatPattern() { return dateTimeFormatPattern; }
public void setDateTimeFormatPattern(String dateTimeFormatPattern) { this.dateTimeFormatPattern = dateTimeFormatPattern; }
public boolean isNew() { return isNew; }
public void setNew(boolean isNew) { this.isNew = isNew; }
public boolean isDeleted() { return deleted; }
public void setDeleted(boolean deleted) { this.deleted = deleted; }
public String getUpdatePeriod() { return updatePeriod; }
public void setUpdatePeriod(String updatePeriod) { this.updatePeriod = updatePeriod; }
public List<String> getPatterns() { return patterns; }
public void setPatterns(List<String> patterns) { this.patterns = patterns; }
public List<String> getParamValues(String param) {
return params==null ? null : this.params.stream()
.filter(p -> p.getParam().equals(param))
.map(EndpointParam::getValue)
.collect(Collectors.toList());
}
public String getSingleParamValue(String param) {
List<String> matchingParamValues = this.getParamValues(param);
return matchingParamValues==null ? null : matchingParamValues.get(matchingParamValues.size()-1);
}
public EndpointParam getSingleParam(String param) {
List<EndpointParam> matchingParams = this.getParams(param);
return matchingParams==null ? null : matchingParams.get(matchingParams.size()-1);
}
}
\ No newline at end of file
package eu.dariah.de.search.model;
import java.io.Serializable;
import lombok.Data;
@Data
public class EndpointParam implements Serializable {
private static final long serialVersionUID = 3155992336436736577L;
private String param;
private String value;
}
......@@ -3,7 +3,9 @@ package eu.dariah.de.search.pojo;
import java.util.List;
import de.unibamberg.minf.dme.model.base.Identifiable;
import lombok.Data;
@Data
public class EndpointPojo implements Identifiable {
private static final long serialVersionUID = 2477306501768367276L;
......@@ -11,8 +13,7 @@ public class EndpointPojo implements Identifiable {
private String url;
private String accessType;
private String fileType;
private String set;
private List<DatasetPojo> datasetPojos;
private boolean unprocessed;
......@@ -20,35 +21,4 @@ public class EndpointPojo implements Identifiable {
private boolean error;
private boolean unaccessible;
public String getId() { return id; }
public void setId(String id) { this.id = id; }
public String getUrl() { return url; }
public void setUrl(String url) { this.url = url; }
public String getAccessType() { return accessType; }
public void setAccessType(String accessType) { this.accessType = accessType; }
public String getFileType() { return fileType; }
public void setFileType(String fileType) { this.fileType = fileType; }
public String getSet() { return set; }
public void setSet(String set) { this.set = set; }
public List<DatasetPojo> getDatasetPojos() { return datasetPojos; }
public void setDatasetPojos(List<DatasetPojo> datasetPojos) { this.datasetPojos = datasetPojos; }
public boolean isUnprocessed() { return unprocessed; }
public void setUnprocessed(boolean unprocessed) { this.unprocessed = unprocessed; }
public boolean isDeleted() { return deleted; }
public void setDeleted(boolean deleted) { this.deleted = deleted; }
public boolean isError() { return error; }
public void setError(boolean error) { this.error = error; }
public boolean isUnaccessible() { return unaccessible; }
public void setUnaccessible(boolean unaccessible) { this.unaccessible = unaccessible; }
}
\ No newline at end of file
......@@ -27,7 +27,6 @@ public class EndpointConverter extends BaseConverter<Endpoint, EndpointPojo> {
ePojo.setUnprocessed(endpoint.isNew());
ePojo.setAccessType(endpoint.getAccessType());
ePojo.setFileType(endpoint.getFileType());
ePojo.setSet(endpoint.getSet());
ePojo.setUrl(endpoint.getUrl());
ePojo.setUnaccessible(endpoint.isUnaccessible());
ePojo.setError(endpoint.isError());
......
......@@ -109,7 +109,7 @@ public class SruQueryExecutionServiceImpl extends BaseResultService implements I
processingSvc.setRoot((Nonterminal)r);
processingSvc.init();
URIBuilder b = new URIBuilder(sruEntry.getValue().getUrl());
URIBuilder b = new URIBuilder(sruEntry.getValue().renderAccessUrl());
b.addParameter("query", ((SimpleQuery)q).getQueryString());
b.addParameter("recordSchema", alias);
b.addParameter("maximumRecords", "10");
......
......@@ -14,7 +14,6 @@ import org.springframework.context.annotation.Scope;
import de.unibamberg.minf.gtf.GtfElementProcessor;
import de.unibamberg.minf.gtf.GtfMappingProcessor;
import de.unibamberg.minf.gtf.extensions.file.downloads.FileDownloader;
import de.unibamberg.minf.mapping.service.MappingExecutionServiceImpl;
import de.unibamberg.minf.processing.git.adapter.GitRepositoryAdapterImpl;
import de.unibamberg.minf.processing.service.json.JsonProcessingService;
......@@ -28,6 +27,7 @@ import eu.dariah.de.search.crawling.crawler.FileProcessor;
import eu.dariah.de.search.crawling.crawler.GitCrawlerImpl;
import eu.dariah.de.search.crawling.crawler.IndexCleaner;
import eu.dariah.de.search.crawling.crawler.OaiPmhCrawlerImpl;
import eu.dariah.de.search.crawling.files.FileDownloader;
import eu.dariah.de.search.crawling.files.FileUnarchiver;
import eu.dariah.de.search.crawling.files.FileUnpacker;
import eu.dariah.de.search.crawling.files.XmlChunker;
......
Subproject commit ebecd6cb2206badd33186021b1f125a966f8f5ad
Subproject commit 85b67ab8ee09fdac92ab1a5baa3d42544bf73608
Subproject commit 951515e27390c580ec6eb5eaa7c38ca28df95f94
Subproject commit b8d42adc16ac531aee20efbd8576c9cfea5948a1
......@@ -11,7 +11,7 @@ includeBuild('../colreg') {
}
}*/
includeBuild('../processing') {
/*includeBuild('../processing') {
dependencySubstitution {
substitute module('de.unibamberg.minf.processing:processing-core') with project(':processing-core')
substitute module('de.unibamberg.minf.processing:processing-adapters') with project(':processing-adapters')
......@@ -35,4 +35,4 @@ includeBuild('../core') {
dependencySubstitution {
substitute module('de.unibamberg.minf.core:core-web') with project(':core-web')
}
}
\ No newline at end of file
}*/
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment