treshenkov/clearing
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xml-importer updated

Browse source
This commit is contained in:
Ivan Nikolaev-Axenov 2024-11-20 17:51:27 +03:00
parent e9fe149587
commit 70036d214b
9 changed files with 107 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/SFTPConfig.java
View file

@ -31,13 +31,13 @@ public class SFTPConfig {
private final Logger log = LoggerFactory.getLogger(getClass());
@Bean("sftpSessionFactorySdf")
@ConditionalOnProperty(value = "import-xml-service.process-sdf-files", havingValue = "true")
@ConditionalOnProperty(value = "import-xml-service.process-files", havingValue = "true")
public SessionFactory<ChannelSftp.LsEntry> sftpSessionFactorySdf(ImportXMLServiceSettings settings) {
DefaultSftpSessionFactory factory = new DefaultSftpSessionFactory(true);
factory.setHost(settings.getStoreSdf().getSftpIn().getServerIp());
factory.setPort(settings.getStoreSdf().getSftpIn().getServerPort());
factory.setUser(settings.getStoreSdf().getSftpIn().getUser());
factory.setPassword(settings.getStoreSdf().getSftpIn().getPassword());
factory.setHost(settings.getStore().getSftpIn().getServerIp());
factory.setPort(settings.getStore().getSftpIn().getServerPort());
factory.setUser(settings.getStore().getSftpIn().getUser());
factory.setPassword(settings.getStore().getSftpIn().getPassword());
factory.setAllowUnknownKeys(true);
return new CachingSessionFactory<>(factory);
}
@ -66,7 +66,7 @@ public class SFTPConfig {
}
@Bean("xmlGatewaySdf")
@ConditionalOnProperty(value = "import-xml-service.process-sdf-files", havingValue = "true")
@ConditionalOnProperty(value = "import-xml-service.process-files", havingValue = "true")
public AnnotationGatewayProxyFactoryBean xmlGatewaySdf() {
return new AnnotationGatewayProxyFactoryBean(XmlGatewaySdf.class);
}
@ -78,7 +78,7 @@ public class SFTPConfig {
}
@Bean("listSftpChannelSdf")
@ConditionalOnProperty(value = "import-xml-service.process-sdf-files", havingValue = "true")
@ConditionalOnProperty(value = "import-xml-service.process-files", havingValue = "true")
public MessageChannel listSftpChannelSdf(@Qualifier("sftpSessionFactorySdf") SessionFactory<ChannelSftp.LsEntry> sessionFactory,
ImportXMLServiceSettings settings) {
DirectChannel dc = new DirectChannel();
@ -96,12 +96,12 @@ public class SFTPConfig {
}
@Bean
@ConditionalOnProperty(value = "import-xml-service.process-sdf-files", havingValue = "true")
@ConditionalOnProperty(value = "import-xml-service.process-files", havingValue = "true")
@ServiceActivator(inputChannel = "listSftpChannelSdf")
public MessageHandler handlerListSdf(@Qualifier("sftpSessionFactorySdf") SessionFactory<ChannelSftp.LsEntry> sessionFactory,
ImportXMLServiceSettings settings) {
SftpOutboundGateway sftpOutboundGateway = new SftpOutboundGateway(sessionFactory, MGET.getCommand(), null);
sftpOutboundGateway.setLocalDirectory(new File(settings.getStoreSdf().getSrcDir()));
sftpOutboundGateway.setLocalDirectory(new File(settings.getStore().getSrcDir()));
sftpOutboundGateway.setAutoCreateLocalDirectory(true);
sftpOutboundGateway.setOption(AbstractRemoteFileOutboundGateway.Option.DELETE);
return sftpOutboundGateway;

2
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/SFTPMockConfig.java
View file

@ -12,7 +12,7 @@ import org.springframework.context.annotation.Profile;
@Configuration
public class SFTPMockConfig {
@Bean("xmlGatewaySdf")
@ConditionalOnProperty(value = "import-xml-service.process-sdf-files", havingValue = "true")
@ConditionalOnProperty(value = "import-xml-service.process-files", havingValue = "true")
public SFTPConfig.XmlGatewaySdf xmlGatewaySdf(){
return new XGatewaySdf();
}

2
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/XMLImporterConfig.java
View file

@ -60,7 +60,7 @@ public class XMLImporterConfig {
}
@Bean("mapOfTable")
@ConditionalOnProperty(value = "import-xml-service.process-sdf-files", havingValue = "true")
@ConditionalOnProperty(value = "import-xml-service.process-files", havingValue = "true")
public Map<ETable, ImdgHazelcast<? extends SpcexObjectBase>> getMapOfTables() {
Map<ETable, ImdgHazelcast<? extends SpcexObjectBase>> map = new HashMap<>();
map.put(ETable.DF_01, (ImdgHazelcast<? extends SpcexObjectBase>) hazelcastService.getImdg(IMDGDistributedNames.Map_SDf01, SDf01.class));

20
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/config/settings/ImportXMLServiceSettings.java
View file

@ -20,7 +20,7 @@ public class ImportXMLServiceSettings {
private KafkaConsumerSettings kafkaConsumer;
@NestedConfigurationProperty
private StoreSettings storeSdf;
private StoreSettings store;
@NestedConfigurationProperty
private StoreSettings storeLks;
@ -29,7 +29,7 @@ public class ImportXMLServiceSettings {
@NestedConfigurationProperty
private Cron cron;
private boolean processSdfFiles;
private boolean processFiles;
private boolean processLksFiles;
public HazelcastClientParams getHazelcast() {
@ -56,12 +56,12 @@ public class ImportXMLServiceSettings {
this.kafkaConsumer = kafkaConsumer;
}
public StoreSettings getStoreSdf() {
return storeSdf;
public StoreSettings getStore() {
return store;
}
public void setStoreSdf(StoreSettings storeSdf) {
this.storeSdf = storeSdf;
public void setStore(StoreSettings store) {
this.store = store;
}
public StoreSettings getStoreLks() {
@ -88,12 +88,12 @@ public class ImportXMLServiceSettings {
this.cron = cron;
}
public boolean isProcessSdfFiles() {
return processSdfFiles;
public boolean isProcessFiles() {
return processFiles;
}
public void setProcessSdfFiles(boolean processSdfFiles) {
this.processSdfFiles = processSdfFiles;
public void setProcessFiles(boolean processFiles) {
this.processFiles = processFiles;
}
public boolean isProcessLksFiles() {

6
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/logic/steps/ChangeDirOfFileStage.java
View file

@ -30,16 +30,16 @@ public class ChangeDirOfFileStage {
log.info("uuid {}. Stage: Change directory of file", resultContainer.getUuid());
File srcDir = new File(resultContainer.getXmlTable().getFileType().equals(FileType.SDF) ?
settings.getStoreSdf().getSrcDir() :
settings.getStore().getSrcDir() :
settings.getStoreLks().getSrcDir());
File outDir;
if (resultContainer.getLastStageStatus().equals(StageResult.ERROR)) {
outDir = new File(resultContainer.getXmlTable().getFileType().equals(FileType.SDF) ?
settings.getStoreSdf().getOutDirError() :
settings.getStore().getOutDirError() :
settings.getStoreLks().getOutDirError());
} else {
outDir = new File(resultContainer.getXmlTable().getFileType().equals(FileType.SDF) ?
settings.getStoreSdf().getOutDir() :
settings.getStore().getOutDir() :
settings.getStoreLks().getOutDir());
}
File xmlFile = resultContainer.getXmlFilePath();

35
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/FileChecker.java
View file

@ -12,6 +12,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -26,12 +27,12 @@ import ru.spcex.platform.utils.log.ExceptionUtils;
public class FileChecker {
private final Logger log = LoggerFactory.getLogger(this.getClass());
private final ImportXMLServiceSettings settings;
private final SFTPConfig.XmlGatewaySdf gatewaySdf;
private final SFTPConfig.XmlGatewayLks gatewayLks;
private final Optional<SFTPConfig.XmlGatewaySdf> gatewaySdf;
private final Optional<SFTPConfig.XmlGatewayLks> gatewayLks;
public FileChecker(ImportXMLServiceSettings settings,
SFTPConfig.XmlGatewaySdf gatewaySdf,
SFTPConfig.XmlGatewayLks gatewayLks) {
Optional<SFTPConfig.XmlGatewaySdf> gatewaySdf,
Optional<SFTPConfig.XmlGatewayLks> gatewayLks) {
this.settings = settings;
this.gatewaySdf = gatewaySdf;
this.gatewayLks = gatewayLks;
@ -42,15 +43,15 @@ public class FileChecker {
List<String> srcDir = new ArrayList<>();
if (specificTable == null) {
if (settings.isProcessSdfFiles()) {
srcDir.add(settings.getStoreSdf().getSrcDir());
if (settings.isProcessFiles()) {
srcDir.add(settings.getStore().getSrcDir());
}
if (settings.isProcessLksFiles()) {
srcDir.add(settings.getStoreLks().getSrcDir());
}
} else {
srcDir.add(specificTable.getFileType().equals(FileType.SDF) ?
settings.getStoreSdf().getSrcDir() :
settings.getStore().getSrcDir() :
settings.getStoreLks().getSrcDir());
}
@ -90,7 +91,7 @@ public class FileChecker {
}
public void checkAndLoadSFTP() {
if (settings.isProcessSdfFiles() && (settings.getStoreSdf().getSftpIn().getSftpSrcPayValDir() == null || settings.getStoreSdf().getSftpIn().getSftpSrcPayValDir().isEmpty())) {
if (settings.isProcessFiles() && (settings.getStore().getSftpIn().getSftpSrcPayValDir() == null || settings.getStore().getSftpIn().getSftpSrcPayValDir().isEmpty())) {
log.error("No SFTP scanning directories, need will be adding setting like 'import-xml-service.store-sdf.sftp-in.sftp-src-pay-val-dir.VAL=/VAL' and restart app");
return;
}
@ -100,19 +101,27 @@ public class FileChecker {
}
List<File> files = new ArrayList<>();
if (settings.isProcessSdfFiles()) {
log.trace("Load from SFTP paths: {}", settings.getStoreSdf().getSftpIn().getSftpSrcPayValDir().values());
for (String path : settings.getStoreSdf().getSftpIn().getSftpSrcPayValDir().values()) {
files.addAll(gatewaySdf.listFiles(path));
if (settings.isProcessFiles()) {
log.trace("Load from SFTP paths: {}", settings.getStore().getSftpIn().getSftpSrcPayValDir().values());
gatewaySdf.ifPresent(gateway -> {
for (String path : settings.getStore().getSftpIn().getSftpSrcPayValDir().values()) {
files.addAll(gateway.listFiles(path));
}
});
if (!files.isEmpty())
log.info("Loaded from SFTP SDF files count={}", files.size());
}
if (settings.isProcessLksFiles()) {
log.trace("Load from SFTP paths: {}", settings.getStoreLks().getSftpIn().getSftpSrcPayValDir().values());
gatewayLks.ifPresent(gateway -> {
for (String path : settings.getStoreLks().getSftpIn().getSftpSrcPayValDir().values()) {
files.addAll(gatewayLks.listFiles(path));
files.addAll(gateway.listFiles(path));
}
});
if (!files.isEmpty())
log.info("Loaded from SFTP LKS files count={}", files.size());
}

4
clearing-parent/xml-importer/src/main/java/ru/spcex/clearing/xml/importer/services/XMLImporterService.java
View file

@ -59,9 +59,9 @@ public class XMLImporterService {
this.filesCurrentlyInProcess = new HashSet<>();
}
@Scheduled(cron = "${import-xml-service.cron.check-src-dir-cron}")
@Scheduled(cron = "${import-xml-service.scheduler.check-src-dir-cron}")
public void run() {
if (settings.isProcessSdfFiles() || settings.isProcessLksFiles()) {
if (settings.isProcessFiles() || settings.isProcessLksFiles()) {
processTable(null);
} else {
log.warn("Turn on SDF or LKS file processing in application.properties!");

56
clearing-parent/xml-importer/src/main/resources/application.properties
View file

@ -2,44 +2,44 @@ server.port=8081
server.servlet.context-path=/xml-importer
spring.main.web-application-type=servlet
import-xml-service.process-sdf-files=true
import-xml-service.process-lks-files=true
import-xml-service.process-files=true
import-xml-service.process-lks-files=false
import-xml-service.cron.check-src-dir-cron=* * * * 1 ?
import-xml-service.scheduler.check-src-dir-cron=* * * * 1 ?
import-xml-service.common.encoding-source=cp866
import-xml-service.common.insert-batch-size=100
import-xml-service.common.threads-count=10
# Store directories for SDf
import-xml-service.store-sdf.delete-src-files=false
import-xml-service.store-sdf.src-dir=/opt/clearing/file/xml-importer/
import-xml-service.store-sdf.out-dir=/opt/clearing/file/xml-importer/loaded/
import-xml-service.store-sdf.out-dir-error=/opt/clearing/file/xml-importer/error/
import-xml-service.store.delete-src-files=false
import-xml-service.store.src-dir=/opt/clearing/file/xml-importer/
import-xml-service.store.out-dir=/opt/clearing/file/xml-importer/loaded/
import-xml-service.store.out-dir-error=/opt/clearing/file/xml-importer/error/
# Sftp directories and credentials for SDf
import-xml-service.store-sdf.sftp-in.sftp-src-pay-val-dir.rub=clearing_xml-importer_sftp/rub/
import-xml-service.store-sdf.sftp-in.sftp-src-pay-val-dir.eur=clearing_xml-importer_sftp/eur/
import-xml-service.store-sdf.sftp-in.sftp-src-dir=clearing_xml-importer_sftp
import-xml-service.store-sdf.sftp-in.user=user
import-xml-service.store-sdf.sftp-in.password=*********
import-xml-service.store-sdf.sftp-in.server-ip=127.0.0.1
import-xml-service.store-sdf.sftp-in.server-port=22
import-xml-service.store.sftp-in.sftp-src-pay-val-dir.rub=clearing_xml-importer_sftp/rub/
import-xml-service.store.sftp-in.sftp-src-pay-val-dir.eur=clearing_xml-importer_sftp/eur/
import-xml-service.store.sftp-in.sftp-src-dir=clearing_xml-importer_sftp
import-xml-service.store.sftp-in.user=user
import-xml-service.store.sftp-in.password=*********
import-xml-service.store.sftp-in.server-ip=127.0.0.1
import-xml-service.store.sftp-in.server-port=22
# Store directories for LKS
import-xml-service.store-lks.delete-src-files=false
import-xml-service.store-lks.src-dir=/opt/clearing/file/xml-importer/
import-xml-service.store-lks.out-dir=/opt/clearing/file/xml-importer/loaded/
import-xml-service.store-lks.out-dir-error=/opt/clearing/file/xml-importer/error/
# Sftp directories and credentials for LKS
import-xml-service.store-lks.sftp-in.sftp-src-pay-val-dir.rub=clearing_xml-importer_sftp/rub/
import-xml-service.store-lks.sftp-in.sftp-src-pay-val-dir.eur=clearing_xml-importer_sftp/eur/
import-xml-service.store-lks.sftp-in.sftp-src-dir=clearing_xml-importer_sftp
import-xml-service.store-lks.sftp-in.user=user
import-xml-service.store-lks.sftp-in.password=*********
import-xml-service.store-lks.sftp-in.server-ip=127.0.0.1
import-xml-service.store-lks.sftp-in.server-port=22
## Store directories for LKS
#import-xml-service.store-lks.delete-src-files=false
#import-xml-service.store-lks.src-dir=/opt/clearing/file/xml-importer/
#import-xml-service.store-lks.out-dir=/opt/clearing/file/xml-importer/loaded/
#import-xml-service.store-lks.out-dir-error=/opt/clearing/file/xml-importer/error/
#
## Sftp directories and credentials for LKS
#import-xml-service.store-lks.sftp-in.sftp-src-pay-val-dir.rub=clearing_xml-importer_sftp/rub/
#import-xml-service.store-lks.sftp-in.sftp-src-pay-val-dir.eur=clearing_xml-importer_sftp/eur/
#import-xml-service.store-lks.sftp-in.sftp-src-dir=clearing_xml-importer_sftp
#import-xml-service.store-lks.sftp-in.user=user
#import-xml-service.store-lks.sftp-in.password=*********
#import-xml-service.store-lks.sftp-in.server-ip=127.0.0.1
#import-xml-service.store-lks.sftp-in.server-port=22
# Hazelcast cluster
import-xml-service.hazelcast.cluster-members=10.200.200.181:5701

56
clearing-parent/xml-importer/src/test/resources/application.properties
View file

@ -2,44 +2,44 @@ server.port=8081
server.servlet.context-path=/xml-importer
spring.main.web-application-type=servlet
import-xml-service.process-sdf-files=true
import-xml-service.process-lks-files=true
import-xml-service.process-files=true
import-xml-service.process-lks-files=false
import-xml-service.cron.check-src-dir-cron=* * * * 1 ?
import-xml-service.scheduler.check-src-dir-cron=* * * * 1 ?
import-xml-service.common.encoding-source=cp866
import-xml-service.common.insert-batch-size=100
import-xml-service.common.threads-count=10
# Store directories for SDf
import-xml-service.store-sdf.delete-src-files=false
import-xml-service.store-sdf.src-dir=/opt/clearing/file/xml-importer/
import-xml-service.store-sdf.out-dir=/opt/clearing/file/xml-importer/loaded/
import-xml-service.store-sdf.out-dir-error=/opt/clearing/file/xml-importer/error/
import-xml-service.store.delete-src-files=false
import-xml-service.store.src-dir=/opt/clearing/file/xml-importer/
import-xml-service.store.out-dir=/opt/clearing/file/xml-importer/loaded/
import-xml-service.store.out-dir-error=/opt/clearing/file/xml-importer/error/
# Sftp directories and credentials for SDf
import-xml-service.store-sdf.sftp-in.sftp-src-pay-val-dir.rub=clearing_xml-importer_sftp/rub/
import-xml-service.store-sdf.sftp-in.sftp-src-pay-val-dir.eur=clearing_xml-importer_sftp/eur/
import-xml-service.store-sdf.sftp-in.sftp-src-dir=clearing_xml-importer_sftp
import-xml-service.store-sdf.sftp-in.user=user
import-xml-service.store-sdf.sftp-in.password=*********
import-xml-service.store-sdf.sftp-in.server-ip=127.0.0.1
import-xml-service.store-sdf.sftp-in.server-port=22
import-xml-service.store.sftp-in.sftp-src-pay-val-dir.rub=clearing_xml-importer_sftp/rub/
import-xml-service.store.sftp-in.sftp-src-pay-val-dir.eur=clearing_xml-importer_sftp/eur/
import-xml-service.store.sftp-in.sftp-src-dir=clearing_xml-importer_sftp
import-xml-service.store.sftp-in.user=user
import-xml-service.store.sftp-in.password=*********
import-xml-service.store.sftp-in.server-ip=127.0.0.1
import-xml-service.store.sftp-in.server-port=22
# Store directories for LKS
import-xml-service.store-lks.delete-src-files=false
import-xml-service.store-lks.src-dir=/opt/clearing/file/xml-importer/
import-xml-service.store-lks.out-dir=/opt/clearing/file/xml-importer/loaded/
import-xml-service.store-lks.out-dir-error=/opt/clearing/file/xml-importer/error/
# Sftp directories and credentials for LKS
import-xml-service.store-lks.sftp-in.sftp-src-pay-val-dir.rub=clearing_xml-importer_sftp/rub/
import-xml-service.store-lks.sftp-in.sftp-src-pay-val-dir.eur=clearing_xml-importer_sftp/eur/
import-xml-service.store-lks.sftp-in.sftp-src-dir=clearing_xml-importer_sftp
import-xml-service.store-lks.sftp-in.user=user
import-xml-service.store-lks.sftp-in.password=*********
import-xml-service.store-lks.sftp-in.server-ip=127.0.0.1
import-xml-service.store-lks.sftp-in.server-port=22
## Store directories for LKS
#import-xml-service.store-lks.delete-src-files=false
#import-xml-service.store-lks.src-dir=/opt/clearing/file/xml-importer/
#import-xml-service.store-lks.out-dir=/opt/clearing/file/xml-importer/loaded/
#import-xml-service.store-lks.out-dir-error=/opt/clearing/file/xml-importer/error/
#
## Sftp directories and credentials for LKS
#import-xml-service.store-lks.sftp-in.sftp-src-pay-val-dir.rub=clearing_xml-importer_sftp/rub/
#import-xml-service.store-lks.sftp-in.sftp-src-pay-val-dir.eur=clearing_xml-importer_sftp/eur/
#import-xml-service.store-lks.sftp-in.sftp-src-dir=clearing_xml-importer_sftp
#import-xml-service.store-lks.sftp-in.user=user
#import-xml-service.store-lks.sftp-in.password=*********
#import-xml-service.store-lks.sftp-in.server-ip=127.0.0.1
#import-xml-service.store-lks.sftp-in.server-port=22
# Hazelcast cluster
import-xml-service.hazelcast.cluster-members=10.200.200.181:5701