treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xml-exporter pipelines removed, localdatetime format fixed

Browse source
This commit is contained in:
Ivan Nikolaev-Axenov 2024-05-28 11:31:42 +03:00
parent 7a157e9ae2
commit e2d043e322
10 changed files with 62 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
clearing-parent/xml-exporter/pom.xml
View file

@ -66,10 +66,6 @@
<groupId>ru.spcex.clearing</groupId>
<artifactId>classes</artifactId>
</dependency>
<dependency>
<groupId>ru.spcex.clearing</groupId>
<artifactId>classes</artifactId>
</dependency>
<!-- special logging -->
<dependency>

18
clearing-parent/xml-exporter/src/main/java/ru/spcex/clearing/xml/exporter/config/XMLExporterConfig.java
View file

@ -8,13 +8,10 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper;
import com.fasterxml.jackson.dataformat.xml.ser.ToXmlGenerator;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLOutputFactory;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
@ -28,27 +25,12 @@ import ru.spcex.clearing.xml.exporter.logic.data.tags.objects.DF53ObjectTag;
import ru.spcex.clearing.xml.exporter.logic.data.tags.objects.DF54ObjectTag;
import ru.spcex.clearing.xml.exporter.logic.data.tags.objects.DF56ObjectTag;
import ru.spcex.clearing.xml.exporter.logic.data.tags.objects.ObjectTag;
import ru.spcex.clearing.xml.exporter.logic.stages.ExportFromHazelcast;
import ru.spcex.clearing.xml.exporter.logic.stages.Journal;
import ru.spcex.clearing.xml.exporter.logic.stages.PrepareXMLFile;
import ru.spcex.clearing.xml.exporter.logic.stages.Stage;
import ru.spcex.platform.classes.base.SpcexObjectBase;
@Configuration
@EnableConfigurationProperties
@ComponentScan(basePackages = {"ru.spcex.clearing.xml.exporter"})
public class XMLExporterConfig {
@Bean("pipeline")
public List<Stage> pipeline(ApplicationContext context) {
List<Stage> pipeline = new LinkedList<>();
pipeline.add(context.getBean(PrepareXMLFile.class));
pipeline.add(context.getBean(ExportFromHazelcast.class));
pipeline.add(context.getBean(Journal.class));
return pipeline;
}
@Bean("mapOfTable")
public Map<Table, ObjectTag<? extends SpcexObjectBase>> getMapOfTables() {
Map<Table, ObjectTag<? extends SpcexObjectBase>> map = new HashMap<>();

40
clearing-parent/xml-exporter/src/main/java/ru/spcex/clearing/xml/exporter/logic/Processor.java
View file

@ -1,40 +0,0 @@
package ru.spcex.clearing.xml.exporter.logic;
import java.util.Arrays;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.xml.exporter.logic.data.ResultContainer;
import ru.spcex.clearing.xml.exporter.logic.data.enums.StageResult;
import ru.spcex.clearing.xml.exporter.logic.stages.Stage;
@Component("processor")
public class Processor {
private final Logger log = LoggerFactory.getLogger(getClass());
private final List<Stage> pipeline;
public Processor(@Qualifier("pipeline") List<Stage> pipeline) {
this.pipeline = pipeline;
}
public void process(ResultContainer task) {
log.info("uuid {}. Task started", task.getUuid());
StageResult result = null;
long startMills = System.currentTimeMillis();
for (Stage currStage : pipeline) {
if (result != null
&& Arrays.asList(StageResult.ERROR, StageResult.COMPLETE).contains(result)
&& currStage.skipCompleted()) continue;
log.info("uuid {}. Stage: {}", task.getUuid(), currStage.getClass().getSimpleName());
result = currStage.process(task);
task.setLastStageResult(result);
}
long endMills = System.currentTimeMillis();
log.info("uuid {}. Task completed, result: {}, time working: {} ms",
task.getUuid(),
result,
endMills - startMills);
}
}

7
clearing-parent/xml-exporter/src/main/java/ru/spcex/clearing/xml/exporter/logic/data/tags/objects/DF51ObjectTag.java
View file

@ -6,6 +6,7 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
import org.springframework.format.annotation.DateTimeFormat;
import ru.clearing.classes.statics.data.sdf.SDf51;
public class DF51ObjectTag extends ObjectTag<SDf51> {
@ -16,7 +17,8 @@ public class DF51ObjectTag extends ObjectTag<SDf51> {
private Long unixtime;
@JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT")
@JsonFormat(pattern = "yyyy-MM-ddhh:mm:ss")
@DateTimeFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss")
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss")
private LocalDateTime datetime;
public DF51ObjectTag() {
@ -25,8 +27,7 @@ public class DF51ObjectTag extends ObjectTag<SDf51> {
@Override
public void setData(SDf51 entity) {
// todo: test datetime formatter
final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-ddhh:mm:ss");
final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss");
this.number = entity.getNumber();
this.unixtime = LocalDateTime.parse(entity.getDatetime(), dtf)
.atZone(ZoneId.systemDefault())

10
clearing-parent/xml-exporter/src/main/java/ru/spcex/clearing/xml/exporter/logic/data/tags/objects/DF56ObjectTag.java
View file

@ -6,6 +6,7 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Objects;
import org.springframework.format.annotation.DateTimeFormat;
import ru.clearing.classes.statics.data.sdf.SDf56;
public class DF56ObjectTag extends ObjectTag<SDf56> {
@ -16,14 +17,16 @@ public class DF56ObjectTag extends ObjectTag<SDf56> {
private Long startUnixtime;
@JacksonXmlProperty(isAttribute = true, localName = "SDATETIME")
@JsonFormat(pattern = "yyyy-MM-ddhh:mm:ss")
@DateTimeFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss")
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss")
private LocalDateTime startDatetime;
@JacksonXmlProperty(isAttribute = true, localName = "EUNIXTIME")
private Long endUnixtime;
@JacksonXmlProperty(isAttribute = true, localName = "EDATETIME")
@JsonFormat(pattern = "yyyy-MM-ddhh:mm:ss")
@DateTimeFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss")
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss")
private LocalDateTime endDatetime;
@JacksonXmlProperty(isAttribute = true, localName = "ACCOUNT")
@ -39,8 +42,7 @@ public class DF56ObjectTag extends ObjectTag<SDf56> {
@Override
public void setData(SDf56 entity) {
// todo: check datatime formatter
final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-ddhh:mm:ss");
final DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss");
this.number = entity.getNumber();
this.startUnixtime = LocalDateTime.parse(entity.getStart_datetime(), dtf)

7
clearing-parent/xml-exporter/src/main/java/ru/spcex/clearing/xml/exporter/logic/stages/ExportFromHazelcast.java
View file

@ -14,6 +14,8 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Supplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import ru.spcex.clearing.platform.messaging.domain.cud.utilities.PairSdfRequest;
@ -33,7 +35,8 @@ import ru.spcex.platform.imdg.api.Imdg;
import ru.spcex.platform.imdg.api.ImdgProvider;
@Component
public class ExportFromHazelcast extends Stage {
public class ExportFromHazelcast {
private final Logger log = LoggerFactory.getLogger(getClass());
private final ExportXMLServiceSettings settings;
private final ImdgProvider imdgProvider;
private final SFTPConfig.XmlGateway gateway;
@ -55,8 +58,8 @@ public class ExportFromHazelcast extends Stage {
this.kafkaSender = kafkaSender;
}
@Override
public StageResult process(ResultContainer resultContainer) {
log.info("uuid {}. Stage: {}", resultContainer.getUuid(), this.getClass().getSimpleName());
Objects.requireNonNull(resultContainer.getTableForExport());
Objects.requireNonNull(resultContainer.getFileForExport());
Table table = resultContainer.getTableForExport();

9
clearing-parent/xml-exporter/src/main/java/ru/spcex/clearing/xml/exporter/logic/stages/Journal.java
View file

@ -18,7 +18,7 @@ import ru.spcex.clearing.xml.exporter.logic.data.enums.StageResult;
import ru.spcex.clearing.xml.exporter.logic.data.enums.Table;
@Component
public class Journal extends Stage implements InitializingBean {
public class Journal implements InitializingBean {
private final Logger log = LoggerFactory.getLogger(getClass());
private final Supplier<KafkaSender> kafkaSender;
@ -27,13 +27,8 @@ public class Journal extends Stage implements InitializingBean {
this.kafkaSender = kafkaSender;
}
@Override
public boolean skipCompleted() {
return false;
}
@Override
public StageResult process(ResultContainer resultContainer) {
log.info("uuid {}. Stage: {}", resultContainer.getUuid(), this.getClass().getSimpleName());
if (kafkaSender == null) {
log.info("kafka producer settings missing; kafka messages to journal-service are not enabled.");
return StageResult.COMPLETE;

9
clearing-parent/xml-exporter/src/main/java/ru/spcex/clearing/xml/exporter/logic/stages/PrepareXMLFile.java
View file

@ -10,6 +10,8 @@ import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.integration.sftp.session.SftpFileInfo;
@ -22,7 +24,8 @@ import ru.spcex.clearing.xml.exporter.logic.data.enums.StageResult;
import ru.spcex.clearing.xml.exporter.logic.data.enums.Table;
@Component
public class PrepareXMLFile extends Stage implements InitializingBean {
public class PrepareXMLFile implements InitializingBean {
private final Logger log = LoggerFactory.getLogger(getClass());
public static final String SECTION = "S";
private final ExportXMLServiceSettings settings;
public static String outDir;
@ -34,10 +37,10 @@ public class PrepareXMLFile extends Stage implements InitializingBean {
this.gateway = gateway;
}
@Override
public StageResult process(ResultContainer resultContainer) {
log.info("uuid {}. Stage: {}", resultContainer.getUuid(), this.getClass().getSimpleName());
Objects.requireNonNull(resultContainer.getTableForExport());
if (settings.getStore().getOutPayValDir() == null || settings.getStore().getOutPayValDir().size() == 0) {
if (settings.getStore().getOutPayValDir() == null || settings.getStore().getOutPayValDir().isEmpty()) {
log.error("No SFTP scanning directories, need will be adding settings like 'export-xml-service.store.out-pay-val-dir.RUB=/RUB' and restart app");
return StageResult.ERROR;
}

16
clearing-parent/xml-exporter/src/main/java/ru/spcex/clearing/xml/exporter/logic/stages/Stage.java
View file

@ -1,16 +0,0 @@
package ru.spcex.clearing.xml.exporter.logic.stages;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ru.spcex.clearing.xml.exporter.logic.data.ResultContainer;
import ru.spcex.clearing.xml.exporter.logic.data.enums.StageResult;
public abstract class Stage {
protected Logger log = LoggerFactory.getLogger(getClass());
public abstract StageResult process(ResultContainer resultContainer);
public boolean skipCompleted() {
return true;
}
}

50
clearing-parent/xml-exporter/src/main/java/ru/spcex/clearing/xml/exporter/services/CommandService.java
View file

@ -13,25 +13,30 @@ import ru.spcex.clearing.platform.messaging.domain.cud.balance.ExportToFileReque
import ru.spcex.clearing.platform.messaging.domain.cud.clearing.SdfClearingRequest;
import ru.spcex.clearing.platform.messaging.domain.cud.schedule.LauncherCommandRequest;
import ru.spcex.clearing.platform.messaging.service.QueueConsumer;
import ru.spcex.clearing.xml.exporter.logic.Processor;
import ru.spcex.clearing.xml.exporter.logic.data.ResultContainer;
import ru.spcex.clearing.xml.exporter.logic.data.enums.StageResult;
import ru.spcex.clearing.xml.exporter.logic.data.enums.Table;
import ru.spcex.clearing.xml.exporter.logic.stages.ExportFromHazelcast;
import ru.spcex.clearing.xml.exporter.logic.stages.Journal;
import ru.spcex.clearing.xml.exporter.logic.stages.PrepareXMLFile;
import ru.spcex.platform.enumeration.Task;
import ru.spcex.platform.imdg.api.ImdgProvider;
@Service
public class CommandService extends QueueConsumer implements InitializingBean {
private final Logger log = LoggerFactory.getLogger(getClass());
private final ImdgProvider imdgProvider;
private final Processor processor;
private final PrepareXMLFile prepareXMLFile;
private final ExportFromHazelcast exportFromHazelcast;
private final Journal journal;
public CommandService(Consumer<String, Object> kafkaQueue,
ImdgProvider imdgProvider,
Processor processor) {
PrepareXMLFile prepareXMLFile,
ExportFromHazelcast exportFromHazelcast,
Journal journal) {
super(kafkaQueue);
this.imdgProvider = imdgProvider;
this.processor = processor;
this.prepareXMLFile = prepareXMLFile;
this.exportFromHazelcast = exportFromHazelcast;
this.journal = journal;
}
@Override
@ -81,7 +86,7 @@ public class CommandService extends QueueConsumer implements InitializingBean {
ResultContainer resultContainer = ResultContainer.createNewTask(tableForExport.get());
resultContainer.setGroupId(request.getSdfGroupId());
resultContainer.setSourceName(request.getFileName());
processor.process(resultContainer);
runProcess(resultContainer);
}
private void processSpecial(Table table, BaseRequest<SdfClearingRequest> systemRequest) {
@ -89,7 +94,7 @@ public class CommandService extends QueueConsumer implements InitializingBean {
ResultContainer resultContainer = ResultContainer.createNewTask(table);
resultContainer.setGroupId(request.getGroupId());
resultContainer.setSourceName(null);
processor.process(resultContainer);
runProcess(resultContainer);
}
private void processByLauncher(Table table, BaseRequest<LauncherCommandRequest> systemRequest) {
@ -98,6 +103,29 @@ public class CommandService extends QueueConsumer implements InitializingBean {
ResultContainer resultContainer = ResultContainer.createNewTask(table);
resultContainer.setGroupId(null);
resultContainer.setSourceName(null);
processor.process(resultContainer);
runProcess(resultContainer);
}
private void runProcess(ResultContainer resultContainer) {
log.info("uuid {}. Task started", resultContainer.getUuid());
StageResult result;
long startMills = System.currentTimeMillis();
result = prepareXMLFile.process(resultContainer);
resultContainer.setLastStageResult(result);
if (result == StageResult.OK) {
result = exportFromHazelcast.process(resultContainer);
resultContainer.setLastStageResult(result);
}
if (result == StageResult.OK) {
result = journal.process(resultContainer);
resultContainer.setLastStageResult(result);
}
long endMills = System.currentTimeMillis();
log.info("uuid {}. Task completed, result: {}, time working: {} ms",
resultContainer.getUuid(),
result,
endMills - startMills);
}
}