Commit f362efe7 authored by Gradl, Tobias's avatar Gradl, Tobias
Browse files

417: Fix crawling HTML log (OPENED)

Task-Url: #417
parent 5f133993
Pipeline #23231 failed with stage
in 10 seconds
......@@ -33,7 +33,9 @@ public class Constants {
public static final String PRESENTATION_FILTERS_KEY = RootElementKeys.PRESENTATION + ".Presentation.Filters";
public static final String ADMINISTRATOR = "ROLE_ADMINISTRATOR";
public static final String CONTRIBUTOR = "ROLE_CONTRIBUTOR";
public static final String USER = "ROLE_USER";
public static final String ES_ELEMENT_CONTAINER_KEY = "properties";
......@@ -78,7 +80,7 @@ public class Constants {
colregName = m;
}
public boolean equalsName(String otherName) {
return (otherName == null) ? false : colregName.equals(otherName);
return otherName != null && colregName.equals(otherName);
}
@Override
......@@ -91,14 +93,14 @@ public class Constants {
CONTENT ("_content"),
INTEGRATIONS ("_integrations"),
PRESENTATION ("_presentation"),
META ("_meta");
META (ELEMENT_KEY_META);
private final String key;
private RootElementKeys(String k) {
key = k;
}
public boolean equalsName(String otherName) {
return (otherName == null) ? false : key.equals(otherName);
return otherName != null && key.equals(otherName);
}
public String toString() {
......@@ -116,7 +118,7 @@ public class Constants {
this.elementName = elementName;
}
public boolean equalsName(String otherName) {
return (otherName == null) ? false : elementName.equals(otherName);
return otherName != null && elementName.equals(otherName);
}
public String toString() {
return this.elementName;
......
package eu.dariah.de.search.config;
import eu.dariah.de.search.config.nested.IndexingLogConfigProperties;
import lombok.Data;
@Data
public class LogConfigProperties {
private IndexingLogConfigProperties indexing;
private String logFile = "search.log";
private String oldlogSuffix = "-%d{yyyy-MM-dd-HH-mm}-%i.log.gz";
private String dir;
private String pattern = "%d [%thread] %-5level %logger{36}[%line] - %msg%n";
private int maxHistory = 90;
private String totalSizeCap = "1GB";
private String maxFileSize = "10MB";
}
package eu.dariah.de.search.config.nested;
import lombok.Data;
@Data
public class IndexingLogConfigProperties {
private String baseDir;
private String pattern = "%d%level%logger{0}%msg%htmlLink";
private String remoteFilesPrefix = "../remoteFiles";
}
......@@ -16,6 +16,5 @@ public class PathsConfigProperties {
private String pictures;
private String downloads;
private String parseErrors;
private String indexingLogs;
private String mappingLogs;
}
\ No newline at end of file
......@@ -3,11 +3,15 @@ package eu.dariah.de.search.controller;
import static org.springframework.web.bind.annotation.RequestMethod.GET;
import static org.springframework.web.bind.annotation.RequestMethod.POST;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.annotation.Secured;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
......@@ -16,6 +20,7 @@ import org.springframework.web.bind.annotation.ResponseBody;
import de.unibamberg.minf.core.web.pojo.MessagePojo;
import de.unibamberg.minf.core.web.pojo.ModelActionPojo;
import eu.dariah.de.search.Constants;
import eu.dariah.de.search.crawling.CrawlManager;
import eu.dariah.de.search.data.service.DownloadDataService;
import eu.dariah.de.search.model.Crawl;
......@@ -179,6 +184,41 @@ public class CollectionEditorController extends BaseController {
return result;
}
@Secured(Constants.ADMINISTRATOR)
@GetMapping("/cleanupCrawls")
public @ResponseBody ModelActionPojo cleanupCrawls(@PathVariable String cId, @RequestParam String epId, @RequestParam String dmId, Model model, Locale locale) {
Crawl lastOnlineCrawl = null;
List<Crawl> crawls = crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Online, CrawlCompleteFlag.Complete, CrawlErrorFlag.NoError, 1);
if (!crawls.isEmpty()) {
lastOnlineCrawl = crawls.get(0);
List<String> clearCrawlIds = new ArrayList<>();
crawls = crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Offline, CrawlCompleteFlag.Both, CrawlErrorFlag.Both, 0);
for (Crawl c : crawls) {
if (c.getModified().isBefore(lastOnlineCrawl.getModified())) {
clearCrawlIds.add(c.getId());
}
}
this.addCrawlId(clearCrawlIds, crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Both, CrawlCompleteFlag.Incomplete, CrawlErrorFlag.Both, 0));
this.addCrawlId(clearCrawlIds, crawlService.findCrawls(epId, dmId, CrawlOnlineFlag.Both, CrawlCompleteFlag.Both, CrawlErrorFlag.Error, 0));
for (String clearCrawlId : clearCrawlIds) {
crawlService.removeCrawlById(clearCrawlId);
}
}
ModelActionPojo result = new ModelActionPojo();
result.setSuccess(true);
return result;
}
private void addCrawlId(List<String> crawlIds, List<Crawl> crawls) {
for (Crawl c : crawls) {
if (!crawlIds.contains(c.getId())) {
crawlIds.add(c.getId());
}
}
}
private void fillStateModel(String cId, String epId, String dmId, Model model, Locale locale, boolean loadCrawls) {
CollectionPojo c = collectionConverter.convert(collectionService.findById(cId), locale);
......
......@@ -4,17 +4,20 @@ import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dariah.de.search.config.LogConfigProperties;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component
public class LogFileDaoImpl extends BaseFsDao implements LogFileDao {
@Autowired private LogConfigProperties logConfig;
@Override
public String find(String id) {
File searchDir = new File(config.getPaths().getIndexingLogs());
File searchDir = new File(logConfig.getIndexing().getBaseDir());
if (!searchDir.exists()) {
return null;
}
......@@ -33,7 +36,7 @@ public class LogFileDaoImpl extends BaseFsDao implements LogFileDao {
@Override
public void delete(String id) {
File searchDir = new File(config.getPaths().getIndexingLogs());
File searchDir = new File(logConfig.getIndexing().getBaseDir());
if (!searchDir.exists()) {
return;
}
......
package eu.dariah.de.search.config;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
import javax.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import ch.qos.logback.classic.Logger;
import ch.qos.logback.classic.LoggerContext;
import ch.qos.logback.classic.encoder.PatternLayoutEncoder;
import ch.qos.logback.classic.html.HTMLLayout;
import ch.qos.logback.classic.sift.MDCBasedDiscriminator;
import ch.qos.logback.classic.sift.SiftingAppender;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.Appender;
import ch.qos.logback.core.Context;
import ch.qos.logback.core.FileAppender;
import ch.qos.logback.core.encoder.LayoutWrappingEncoder;
import ch.qos.logback.core.helpers.NOPAppender;
import ch.qos.logback.core.joran.spi.JoranException;
import ch.qos.logback.core.rolling.RollingFileAppender;
import ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP;
import ch.qos.logback.core.rolling.TimeBasedRollingPolicy;
import ch.qos.logback.core.sift.AppenderFactory;
import ch.qos.logback.core.status.InfoStatus;
import ch.qos.logback.core.status.StatusManager;
import ch.qos.logback.core.util.FileSize;
import de.unibamberg.minf.core.web.init.logging.CustomHTMLLayout;
import eu.dariah.de.search.config.nested.IndexingLogConfigProperties;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Data
@EqualsAndHashCode(callSuper=true)
@Configuration
@ConfigurationProperties(prefix="log")
public class LogConfig extends LogConfigProperties {
private static final String SIFT_DISCRIMINATOR_DEFAULT_VALUE = "uid_unset";
@Autowired private MainConfig mainConfig;
@PostConstruct
public void reconfigureLogging() throws IOException {
log.debug("Reconfiguring logging...");
if (this.getIndexing()==null) {
this.setIndexing(new IndexingLogConfigProperties());
}
if (this.getIndexing().getBaseDir()==null) {
this.getIndexing().setBaseDir(mainConfig.setupPath(mainConfig.getPaths().getData(), "indexingLogs"));
}
LoggerContext lc = (LoggerContext) org.slf4j.LoggerFactory.getILoggerFactory();
StatusManager statusManager = lc.getStatusManager();
if (statusManager != null) {
statusManager.add(new InfoStatus("Configuring logger", lc));
}
Logger rootLogger = lc.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME);
// RollingFileAppender
if (this.getDir()!=null) {
rootLogger.addAppender(this.configureRollingFileAppender(lc));
}
rootLogger.addAppender(this.configureSiftHtmlAppender(lc));
this.logAppenderConfiguration(rootLogger);
}
private void logAppenderConfiguration(Logger rootLogger) {
log.info("Logging reconfigured");
Iterator<Appender<ILoggingEvent>> appendersIterator = rootLogger.iteratorForAppenders();
Appender<ILoggingEvent> configuredAppender;
while(appendersIterator.hasNext()) {
configuredAppender = appendersIterator.next();
log.info("Configured appender: {}", configuredAppender.getClass().getName());
}
}
private Appender<ILoggingEvent> configureSiftHtmlAppender(Context context) throws IOException {
SiftingAppender sa = new SiftingAppender();
sa.setName("SIFT-HTML");
sa.setContext(context);
MDCBasedDiscriminator discriminator = new MDCBasedDiscriminator();
discriminator.setKey("uid");
discriminator.setDefaultValue(SIFT_DISCRIMINATOR_DEFAULT_VALUE);
discriminator.start();
sa.setDiscriminator(discriminator);
final String logPath = this.setupLogFile(this.getIndexing().getBaseDir(), "{}.html");
final String pattern = this.getIndexing().getPattern();
final String filesPrefix = this.getIndexing().getRemoteFilesPrefix();
log.info("SiftingAppender (SIFT-HTML) logging to base directory: {}", logPath);
sa.setAppenderFactory(new AppenderFactory<ILoggingEvent>() {
@Override
public Appender<ILoggingEvent> buildAppender(Context context, String discriminatingValue) throws JoranException {
// No appending for unset key, as we are not in an indexing process then
if (discriminatingValue.equals(SIFT_DISCRIMINATOR_DEFAULT_VALUE)) {
Appender<ILoggingEvent> nopAppender = new NOPAppender<>();
nopAppender.setContext(context);
return nopAppender;
}
FileAppender<ILoggingEvent> appender = new FileAppender<>();
appender.setFile(logPath.replace("{}", discriminatingValue));
appender.setContext(context);
CustomHTMLLayout layout = new CustomHTMLLayout();
layout.setPattern(pattern);
layout.setContext(context);
layout.setRemoteFilesPrefix(filesPrefix);
LayoutWrappingEncoder<ILoggingEvent> pl = new LayoutWrappingEncoder<>();
pl.setLayout(layout);
pl.setContext(context);
pl.start();
layout.start();
appender.setLayout(layout);
appender.start();
return appender;
}
});
sa.start();
return sa;
}
private Appender<ILoggingEvent> configureRollingFileAppender(Context context) throws IOException {
RollingFileAppender<ILoggingEvent> appender = new RollingFileAppender<>();
appender.setName("FILE");
appender.setContext(context);
appender.setFile(this.setupLogFile(this.getDir(), this.getLogFile()));
TimeBasedRollingPolicy<ILoggingEvent> policy = new TimeBasedRollingPolicy<>();
policy.setContext(context);
policy.setMaxHistory(this.getMaxHistory());
policy.setFileNamePattern(this.setupLogFile(this.getDir(), this.getLogFile() + this.getOldlogSuffix()));
policy.setParent(appender);
policy.start();
SizeAndTimeBasedFNATP<ILoggingEvent> innerpolicy = new SizeAndTimeBasedFNATP<>();
innerpolicy.setContext(context);
innerpolicy.setMaxFileSize(FileSize.valueOf(this.getMaxFileSize()));
innerpolicy.setTimeBasedRollingPolicy(policy);
innerpolicy.start();
policy.setTimeBasedFileNamingAndTriggeringPolicy(innerpolicy);
policy.start();
appender.setRollingPolicy(policy);
PatternLayoutEncoder pl = new PatternLayoutEncoder();
pl.setContext(context);
pl.setPattern(this.getPattern());
pl.start();
appender.setEncoder(pl);
log.info("RollingFileAppender logging to file: {}", appender.getFile());
appender.start();
return appender;
}
private String setupLogFile(String first, String... more) throws IOException {
Path p = Paths.get(first, more);
if (!Files.exists(p.getParent())) {
Files.createDirectories(p.getParent());
}
return p.toString();
}
}
......@@ -3,6 +3,7 @@ package eu.dariah.de.search.config;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.NoSuchAlgorithmException;
......@@ -17,11 +18,9 @@ import org.springframework.core.io.ResourceLoader;
import com.fasterxml.jackson.databind.ObjectMapper;
import de.unibamberg.minf.core.util.json.JsonNodeHelper;
import de.unibamberg.minf.core.web.navigation.Navigation;
import de.unibamberg.minf.core.web.service.ImageServiceImpl.SizeBoundsType;
import eu.dariah.de.search.config.nested.ImagesConfigProperties;
import eu.dariah.de.search.config.nested.ThumbnailsConfigProperties;
import eu.dariah.de.search.crawling.TimedCrawlManagerImpl;
import eu.dariah.de.search.data.service.CachedImageServiceImpl;
import eu.dariah.de.search.exceptions.ConfigurationException;
import eu.dariah.de.search.updates.UpdateServiceImpl;
......@@ -38,56 +37,62 @@ public class MainConfig extends MainConfigProperties {
@Autowired ResourceLoader resourceLoader;
@PostConstruct
public void completeConfiguration() {
public void completeConfiguration() throws IOException {
// Config
if (paths.getConfig()==null) {
paths.setConfig(Paths.get(paths.getMain(), "config").toString());
paths.setConfig(this.setupPath(paths.getMain(), "config"));
}
// Backup
if (paths.getBackups()==null) {
paths.setBackups(Paths.get(paths.getMain(), "backups").toString());
paths.setBackups(this.setupPath(paths.getMain(), "backups"));
}
// Data and subdirs
if (paths.getData()==null) {
paths.setData(Paths.get(paths.getMain(), "data").toString());
paths.setData(this.setupPath(paths.getMain(), "data"));
}
if (paths.getDatamodels()==null) {
paths.setDatamodels(Paths.get(paths.getData(), "datamodels").toString());
paths.setDatamodels(this.setupPath(paths.getData(), "datamodels"));
}
if (paths.getMappings()==null) {
paths.setMappings(Paths.get(paths.getData(), "mappings").toString());
paths.setMappings(this.setupPath(paths.getData(), "mappings"));
}
if (paths.getGrammars()==null) {
paths.setGrammars(Paths.get(paths.getData(), "grammars").toString());
paths.setGrammars(this.setupPath(paths.getData(), "grammars"));
}
if (paths.getModels()==null) {
paths.setModels(Paths.get(paths.getData(), "models").toString());
paths.setModels(this.setupPath(paths.getData(), "models"));
}
if (paths.getCrawls()==null) {
paths.setCrawls(Paths.get(paths.getData(), "crawls").toString());
paths.setCrawls(this.setupPath(paths.getData(), "crawls"));
}
if (paths.getPictures()==null) {
paths.setPictures(Paths.get(paths.getData(), "pictures").toString());
paths.setPictures(this.setupPath(paths.getData(), "pictures"));
}
if (paths.getParseErrors()==null) {
paths.setParseErrors(Paths.get(paths.getData(), "parseErrors").toString());
paths.setParseErrors(this.setupPath(paths.getData(), "parseErrors"));
}
if (paths.getDownloads()==null) {
paths.setDownloads(Paths.get(paths.getData(), "downloads").toString());
paths.setDownloads(this.setupPath(paths.getData(), "downloads"));
}
if (paths.getIndexingLogs()==null) {
paths.setIndexingLogs(Paths.get(paths.getData(), "indexingLogs").toString());
}
if (images==null) {
images = new ImagesConfigProperties();
images.setThumbnails(new ThumbnailsConfigProperties());
}
}
}
protected String setupPath(String first, String... more) throws IOException {
Path p = Paths.get(first, more);
if (!Files.exists(p)) {
Files.createDirectories(p);
}
return p.toString();
}
@Bean
public CachedImageServiceImpl cachedImageService() {
CachedImageServiceImpl cachedImageServiceImpl = new CachedImageServiceImpl();
......
......@@ -48,6 +48,8 @@ public class WebConfig implements WebMvcConfigurer {
private LocalizationConfig localization;
@Autowired private MainConfig mainConfig;
@Autowired private LogConfig logConfig;
@Autowired ResourceLoader resourceLoader;
@PostConstruct
......@@ -95,6 +97,10 @@ public class WebConfig implements WebMvcConfigurer {
registry.addResourceHandler("/resources/**").addResourceLocations("/resources/");
registry.addResourceHandler("/themes/**").addResourceLocations("/themes/");
registry.addResourceHandler("/theme/**").addResourceLocations("/themes/" + theme + "/");
registry.addResourceHandler("/logs/**").addResourceLocations("file://" + logConfig.getIndexing().getBaseDir() + "/");
registry.addResourceHandler("/remoteFiles/**").addResourceLocations("file://" + mainConfig.getPaths().getParseErrors() + "/");
}
/*@Override
......
......@@ -5,13 +5,12 @@ paths:
# grammars: ${paths.main}/grammars
# models: ${paths.main}/models
# config: ${paths.main}/config
# data: ${paths.main}/data
data: ${paths.main}/data
# crawls: ${paths.data}/crawls
# pictures: ${paths.data}/images
# downloads: ${paths.data}/downloads
# backups: ${paths.data}/backups
# parseErrors: ${paths.data}/parseErrors
# indexing_logs: ${paths.data}/log/indexing
# mapping_logs: ${paths.data}/log/mapping
#images:
......@@ -28,6 +27,14 @@ paths:
# username:
# password:
log:
dir: /tmp
# pattern: "%-4relative [%thread] %-5level %logger{35}[%line]- %msg%n"
indexing:
baseDir: /tmp/indexing_logs
indexing:
index_name_prefix: gs_
index_config_file: index_config.json
......@@ -82,7 +89,7 @@ theme: tutorial-finder-theme
mongo:
host: 127.0.0.1
port: 27017
database: colreg
database: search_boot
......
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<conversionRule conversionWord="htmlLink" converterClass="de.unibamberg.minf.core.web.init.logging.LinkConverter" />
<appender name="Console"
class="ch.qos.logback.core.ConsoleAppender">
<layout class="ch.qos.logback.classic.PatternLayout">
<pattern>%d{yyyy-MM-dd} %d{HH:mm:ss.SSS} %thread | %5p %logger{36}.%M\(%line\) - %m%n</pattern>
</layout>
</appender>
<appender name="SIFT-HTML" class="ch.qos.logback.classic.sift.SiftingAppender">
<discriminator>
<Key>uid</Key>
<DefaultValue>uid_unset</DefaultValue>
</discriminator>
<sift>
<appender name="htmlAppender" class="ch.qos.logback.core.FileAppender">
<file>/tmp/test_${uid}.html</file>
<encoder class="ch.qos.logback.core.encoder.LayoutWrappingEncoder">
<layout class="de.unibamberg.minf.core.web.init.logging.CustomHTMLLayout">
<pattern>%d%level%logger{0}%msg%htmlLink</pattern>
<remoteFilesPrefix>../remoteFiles</remoteFilesPrefix>
</layout>
</encoder>
</appender>
</sift>
</appender>
<!-- LOG everything at INFO level -->
<root level="info">
<appender-ref ref="Console" />
</root>
......
Subproject commit 475e93de97114875892a0cd03a33e510f03dddd5
Subproject commit 355208a102f0d3712cfa144a37839f37db5a6b22
Subproject commit 42796570bd36dbc1812c859fb3473b95e5133063
Subproject commit 951515e27390c580ec6eb5eaa7c38ca28df95f94
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!