Added MonitoringSignalResponseStaxXmlReader and MonitoringSignalResponseStaxXmlWriter; Used batches to write response file and update hits.

This commit is contained in:
Mikhail Trofimov 2025-07-18 18:10:48 +03:00
parent f052841954
commit cb5f0d9a9b
10 changed files with 294 additions and 114 deletions

View file

@ -4,7 +4,6 @@ import jakarta.xml.bind.JAXBException;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import ru.nbch.credit_tracker.entities.response.errors.MonitoringError;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.MonitoringSignalResponse;
import ru.nbch.credit_tracker.entities.response.online_report.SgnlOnlineReport;
import ru.nbch.credit_tracker.entities.response.success.MonitoringResponse;
import ru.nbch.credit_tracker.service.xml.jaxb.IXmlProcessor;
@ -25,8 +24,7 @@ public class XmlConfig {
SgnlOnlineReport.class,
// marshalling classes
MonitoringError.class,
MonitoringResponse.class,
MonitoringSignalResponse.class);
MonitoringResponse.class);
}
@Bean
@ -35,6 +33,7 @@ public class XmlConfig {
staxReader.add(new SgnlReportStatusStaxReader());
staxReader.add(new MonitoringRequestStaxReader());
staxReader.add(new OnlineDownloadReportPersonsStaxReader());
staxReader.add(new MonitoringSignalResponseStaxXmlReader());
return new XmlProcessorStaxImpl(staxReader);
}
}

View file

@ -17,5 +17,5 @@ public class MonitoringSignalResponse {
@XmlElement(name = "PackageName", required = true)
protected String packageName;
@XmlElement(name = "Subjects", required = true)
protected Subjects subjects;
protected Subjects subjects = new Subjects();
}

View file

@ -24,4 +24,17 @@ public class Signals {
return this.signals;
}
private Signal currentSignal;
public void createCurrentSignal() {
currentSignal = new Signal();
}
public Signal getCurrentSignal() {
return currentSignal;
}
public void finalizeSignal() {
getSignals().add(currentSignal);
}
}

View file

@ -30,7 +30,7 @@ public class Subject {
@XmlElement(name = "Flag90_12", required = true)
protected Integer flag9012;
@XmlElement(name = "Signals", required = true)
protected Signals signals;
protected Signals signals = new Signals();
}

View file

@ -24,4 +24,18 @@ public class Subjects {
return this.subjects;
}
private Subject currentSubject;
public void createCurrentSubject() {
currentSubject = new Subject();
}
public Subject getCurrentSubject() {
return currentSubject;
}
public void finalizeSubject() {
getSubjects().add(currentSubject);
}
}

View file

@ -8,7 +8,10 @@ import ru.nbch.credit_tracker.config.settings.CreditTrackerSettings;
import ru.nbch.credit_tracker.entities.dto.Hit;
import ru.nbch.credit_tracker.entities.dto.PhoneFidMap;
import ru.nbch.credit_tracker.entities.dto.UserPackage;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.*;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.MonitoringSignalResponse;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.Signal;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.Signals;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.Subject;
import ru.nbch.credit_tracker.entities.response.online_download.Inquiry;
import ru.nbch.credit_tracker.entities.response.online_download.Person;
import ru.nbch.credit_tracker.entities.response.online_download.Persons;
@ -16,11 +19,11 @@ import ru.nbch.credit_tracker.entities.response.online_report.Report;
import ru.nbch.credit_tracker.entities.response.online_report.SgnlOnlineReport;
import ru.nbch.credit_tracker.service.signal.SignalRestService;
import ru.nbch.credit_tracker.service.xml.jaxb.IXmlProcessor;
import ru.nbch.credit_tracker.service.xml.stax.MonitoringSignalResponseStaxXmlWriter;
import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
import ru.nbch.credit_tracker.utils.CTUtil;
import java.io.*;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
@ -38,10 +41,8 @@ public class MonitoringService {
private final SignalRestService signalRestService;
private final SignalService signalService;
private final IXmlProcessor xmlProcessor;
private final StaxXmlProcessor staxXmlProcessor;
private final CreditTrackerSettings config;
private final Integer batchSize = 10000;
public MonitoringService(SignalRestService signalRestService,
SignalService signalService,
@ -50,7 +51,6 @@ public class MonitoringService {
CreditTrackerSettings config) {
this.signalRestService = signalRestService;
this.signalService = signalService;
this.xmlProcessor = xmlProcessor;
this.staxXmlProcessor = staxXmlProcessor;
this.config = config;
}
@ -108,12 +108,11 @@ public class MonitoringService {
continue;
}
MonitoringSignalResponse response = generateResponse(userPackage, reportPath);
String fileName = createResponseFile(userPackage, response);
String responsePath = generateResponse(userPackage, reportPath);
String fileName = createResponseFile(userPackage, responsePath);
if (fileName != null) {
log.info("New report fileName: {}", fileName);
updateSignalHits(userPackage, response, fileName);
updateSignalHits(userPackage, responsePath, fileName);
signalService.updateLastDownloadedReport(userPackage.getId(), newReport.get());
} else {
log.info("Report was not created for package {}", userPackage.getPackageName());
@ -125,16 +124,21 @@ public class MonitoringService {
log.info("Finish monitoring");
}
private MonitoringSignalResponse generateResponse(UserPackage userPackage, String reportPath) {
private String generateResponse(UserPackage userPackage, String reportPath) {
String folderPath = config.getLoadDir() + File.separator + "tmp_folder" + File.separator + userPackage.getId() + File.separator;
String responsePath = folderPath + UUID.randomUUID() + "_response_output.xml";
MonitoringSignalResponse response = new MonitoringSignalResponse();
response.setPackageName(userPackage.getPackageName()); // Имя пакета, переданного при установке
Subjects subjects = new Subjects();
try (InputStream in = Files.newInputStream(Path.of(reportPath))) {
MonitoringSignalResponseStaxXmlWriter staxXmlWriter = new MonitoringSignalResponseStaxXmlWriter("windows-1251");
staxXmlWriter.writeWithStax(responsePath, response,
writer -> {
try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(reportPath)), 4 * 1024 * 1024)) {
staxXmlProcessor.read(in, Persons.class,
personsData -> {
List<Subject> subjectsBatch = processPersons(userPackage, personsData.getPersons());
subjects.getSubjects().addAll(subjectsBatch);
writer.accept(subjectsBatch);
personsData.getPersons().clear();
},
10000,
@ -142,7 +146,7 @@ public class MonitoringService {
} catch (IOException e) {
throw new RuntimeException("Error while parsing nbki downloaded report", e);
}
response.setSubjects(subjects);
});
try {
Files.delete(Path.of(reportPath));
@ -150,7 +154,7 @@ public class MonitoringService {
log.warn("File {} hasn't been deleted", reportPath);
}
return response;
return responsePath;
}
private List<Subject> processPersons(UserPackage userPackage, List<Person> persons) {
@ -159,7 +163,7 @@ public class MonitoringService {
Signals signals = new Signals();
for (Inquiry inquiry : person.getInquiries()) {
// Отбираем только тех фл, у которых bus_category in (MFO, MKK, MFK) и есть сигналы с кодом 16
if (!CTUtil.checkBusCategory(inquiry.getBusCategory()) || inquiry.getSignals().stream().anyMatch(signal -> signal.getN().equals(16))) {
if (!CTUtil.checkBusCategory(inquiry.getBusCategory()) || inquiry.getSignals().stream().noneMatch(signal -> signal.getN().equals(16))) {
continue;
}
Signal signal = new Signal();
@ -186,7 +190,7 @@ public class MonitoringService {
return subjects;
}
private String createResponseFile(UserPackage userPackage, MonitoringSignalResponse response) {
private String createResponseFile(UserPackage userPackage, String responsePath) {
try {
String fileName = CTUtil.generateResponseReportFileName(userPackage.getUserId(), userPackage.getPackageName());
String filePath = config.getLoadDir() + File.separator + userPackage.getUserId().substring(0, 6) + File.separator;
@ -196,27 +200,40 @@ public class MonitoringService {
folder.mkdir();
}
String xmlStr = xmlProcessor.asString(response, true);
byte[] xmlBytes = xmlStr.getBytes(Charset.forName("windows-1251"));
try (FileOutputStream fos = new FileOutputStream(filePath + fileName + ".zip");
ZipOutputStream zos = new ZipOutputStream(fos)) {
try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(responsePath)), 4 * 1024 * 1024);
BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath + fileName + ".zip"));
ZipOutputStream zos = new ZipOutputStream(bos)) {
ZipEntry zipEntry = new ZipEntry(fileName + ".xml");
zos.putNextEntry(zipEntry);
zos.write(xmlBytes);
byte[] buffer = new byte[65536];
int bytesRead;
while ((bytesRead = in.read(buffer)) > 0) {
zos.write(buffer, 0, bytesRead);
}
zos.finish();
zos.closeEntry();
}
return fileName;
} catch (Exception e) {
log.error("Ошибка при записи отчета пользователя в файл: ", e);
}
try {
Files.delete(Path.of(responsePath));
} catch (IOException e) {
log.warn("File {} hasn't been deleted", responsePath);
}
return null;
}
private void updateSignalHits(UserPackage userPackage, MonitoringSignalResponse response, String reportFileName) {
private void updateSignalHits(UserPackage userPackage, String responsePath, String reportFileName) {
log.info("Updating signal hits for {}", userPackage.getPackageName());
LocalDateTime responseSentAt = LocalDateTime.now();
try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(Path.of(responsePath)), 4 * 1024 * 1024)) {
staxXmlProcessor.read(in, MonitoringSignalResponse.class,
response -> {
List<Hit> hits = new ArrayList<>();
response.getSubjects().getSubjects().forEach(subject -> {
subject.getSignals().getSignals().forEach(signal -> {
@ -228,11 +245,22 @@ public class MonitoringService {
hit.setReportFileName(reportFileName);
hits.add(hit);
});
signalService.insertHits(hits);
});
for (int i = 0; i < hits.size(); i += batchSize) {
int end = Math.min(i + batchSize, hits.size());
signalService.insertHits(hits.subList(i, end));
response.getSubjects().getSubjects().clear();
},
10000,
"/MonitoringSignalResponse/Subjects/Subject");
} catch (IOException e) {
throw new RuntimeException("Error while and updating hits", e);
}
try {
Files.delete(Path.of(responsePath));
} catch (IOException e) {
log.warn("File {} hasn't been deleted", responsePath);
}
log.info("Signal hits updated for package {}", userPackage.getPackageName());
}

View file

@ -0,0 +1,83 @@
package ru.nbch.credit_tracker.service.xml.stax;
import lombok.extern.slf4j.Slf4j;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.MonitoringSignalResponse;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
@Slf4j
public class MonitoringSignalResponseStaxXmlReader extends BaseStaxReader<MonitoringSignalResponse>{
@Override
protected MonitoringSignalResponse createObject() {
return new MonitoringSignalResponse();
}
@Override
protected boolean innerProcess(XMLStreamReader reader, MonitoringSignalResponse obj) throws XMLStreamException {
String text = buffer.toString();
String currPath = getPath();
switch (currPath) {
case "/MonitoringSignalResponse/Subjects/Subject" ->
obj.getSubjects().finalizeSubject();
case "/MonitoringSignalResponse/Subjects/Subject/Signals/Signal" ->
obj.getSubjects().getCurrentSubject().getSignals().finalizeSignal();
case "/MonitoringSignalResponse/Subjects/Subject/Id" ->
obj.getSubjects().getCurrentSubject().setId(text);
case "/MonitoringSignalResponse/Subjects/Subject/ClientId" ->
obj.getSubjects().getCurrentSubject().setClientId(text);
case "/MonitoringSignalResponse/Subjects/Subject/FlagDebt5000" -> {
try {
int flagDebt5000 = Integer.parseInt(text);
obj.getSubjects().getCurrentSubject().setFlagDebt5000(flagDebt5000);
} catch (NumberFormatException e) {
log.warn("Couldn't parse flagDebt5000: {}", text);
}
}
case "/MonitoringSignalResponse/Subjects/Subject/Flag90_12" -> {
try {
int flag9012 = Integer.parseInt(text);
obj.getSubjects().getCurrentSubject().setFlag9012(flag9012);
} catch (NumberFormatException e) {
log.warn("Couldn't parse flag9012: {}", text);
}
}
case "/MonitoringSignalResponse/Subjects/Subject/Signals/Signal/SignalType" -> {
try {
int signalType = Integer.parseInt(text);
obj.getSubjects().getCurrentSubject().getSignals().getCurrentSignal().setSignalType(signalType);
} catch (NumberFormatException e) {
log.warn("Couldn't parse signalType: {}", text);
}
}
case "/MonitoringSignalResponse/Subjects/Subject/Signals/Signal/InquiryDate" ->
obj.getSubjects().getCurrentSubject().getSignals().getCurrentSignal().setInquiryDate(text);
case "/MonitoringSignalResponse/Subjects/Subject/Signals/Signal/InquiryPurpose" -> {
try {
int inquiryPurpose = Integer.parseInt(text);
obj.getSubjects().getCurrentSubject().getSignals().getCurrentSignal().setInquiryPurpose(inquiryPurpose);
} catch (NumberFormatException e) {
log.warn("Couldn't parse inquiryPurpose: {}", text);
}
}
}
return false;
}
@Override
protected void innerProcessStartElement(XMLStreamReader reader, MonitoringSignalResponse obj) throws XMLStreamException {
String currPath = getPath();
switch (currPath) {
case "/MonitoringSignalResponse/Subjects/Subject" ->
obj.getSubjects().createCurrentSubject();
case "/MonitoringSignalResponse/Subjects/Subject/Signals/Signal" ->
obj.getSubjects().getCurrentSubject().getSignals().createCurrentSignal();
}
}
@Override
public Class<MonitoringSignalResponse> type() {
return MonitoringSignalResponse.class;
}
}

View file

@ -0,0 +1,89 @@
package ru.nbch.credit_tracker.service.xml.stax;
import lombok.extern.slf4j.Slf4j;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.MonitoringSignalResponse;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.Signal;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.Signals;
import ru.nbch.credit_tracker.entities.response.monitoring_signal_response.Subject;
import javax.xml.stream.XMLEventFactory;
import javax.xml.stream.XMLEventWriter;
import javax.xml.stream.XMLOutputFactory;
import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.util.List;
@Slf4j
public class MonitoringSignalResponseStaxXmlWriter extends StaxXmlWriter<MonitoringSignalResponse, Subject>{
public MonitoringSignalResponseStaxXmlWriter(String charsetName) {
super(charsetName);
}
@Override
public void writeWithStax(String filePath, MonitoringSignalResponse obj, LoaderWriter<Subject> loader) {
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath), 4 * 1024 * 1024)) {
XMLOutputFactory output = XMLOutputFactory.newInstance();
XMLEventFactory xmlEventFactory = XMLEventFactory.newInstance();
XMLEventWriter writer = output.createXMLEventWriter(bos, charsetName);
writer = new IndentingXMLEventWriter(writer);
writerWrapper = new WriterWrapper(xmlEventFactory, writer);
writerWrapper.addStartDocument(charsetName);
writerWrapper.addStartElement("MonitoringSignalResponse");
writerWrapper.addStartElement("Subjects");
loader.accept(subjects -> {
try {
writeSubjects(subjects);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
writerWrapper.addEndElement("Subjects");
writerWrapper.addEndElement("MonitoringSignalResponse");
writerWrapper.addEndDocument();
writerWrapper.flush();
System.gc();
} catch (Throwable e) {
log.error("Error serializing object to XML string: {}", e.getMessage(), e);
throw new IllegalStateException("cannot serialize object to XML");
}
}
private void writeSubjects(List<Subject> subjects) throws Exception {
writerWrapper.writeCollection("Subject", subjects, this::writeSubject);
}
private void writeSubject(Subject subject, String name) throws Exception {
if (subject == null) {
return;
}
writerWrapper.addStartElement(name);
writerWrapper.addSimplelement("Id", subject.getId());
writerWrapper.addSimplelement("ClientId", subject.getClientId());
writerWrapper.addSimplelement("FlagDebt5000", subject.getFlagDebt5000() != null ? String.valueOf(subject.getFlagDebt5000()) : null);
writerWrapper.addSimplelement("Flag90_12", subject.getFlag9012() != null ? String.valueOf(subject.getFlag9012()) : null);
writeSignals( subject.getSignals(), "Signals");
writerWrapper.addEndElement(name);
}
private void writeSignals(Signals signals, String name) throws Exception {
writerWrapper.addStartElement(name);
writerWrapper.writeCollection("Signal", signals.getSignals(), this::writeSignal);
writerWrapper.addEndElement(name);
}
private void writeSignal(Signal signal, String name) throws Exception {
writerWrapper.addStartElement(name);
writerWrapper.addSimplelement("SignalType", String.valueOf(signal.getSignalType()));
writerWrapper.addSimplelement("InquiryDate", signal.getInquiryDate());
writerWrapper.addSimplelement("InquiryPurpose", String.valueOf(signal.getInquiryPurpose()));
writerWrapper.addEndElement(name);
}
}

View file

@ -1,29 +1,29 @@
package ru.nbch.credit_tracker.service.xml.stax;
import lombok.extern.slf4j.Slf4j;
import ru.nbch.credit_tracker.entities.signal_package.*;
import ru.nbch.credit_tracker.entities.signal_package.Auth;
import ru.nbch.credit_tracker.entities.signal_package.Person;
import ru.nbch.credit_tracker.entities.signal_package.Signals;
import javax.xml.stream.XMLEventFactory;
import javax.xml.stream.XMLEventWriter;
import javax.xml.stream.XMLOutputFactory;
import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.OutputStream;
import java.util.List;
import java.util.function.Consumer;
@Slf4j
public class SignalsStaxXmlWriter extends StaxXmlWriter<Signals> {
public class SignalsStaxXmlWriter extends StaxXmlWriter<Signals, Person> {
public SignalsStaxXmlWriter(String charsetName) {
super(charsetName);
}
public void writeWithStax(String filePath, Signals signals, LoaderWriter loader) {
try (FileOutputStream out = new FileOutputStream(filePath)) {
public void writeWithStax(String filePath, Signals signals, LoaderWriter<Person> loader) {
try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath), 4 * 1024 * 1024)) {
XMLOutputFactory output = XMLOutputFactory.newInstance();
XMLEventFactory xmlEventFactory = XMLEventFactory.newInstance();
XMLEventWriter writer = output.createXMLEventWriter(out, charsetName);
XMLEventWriter writer = output.createXMLEventWriter(bos, charsetName);
writer = new IndentingXMLEventWriter(writer);
writerWrapper = new WriterWrapper(xmlEventFactory, writer);
@ -64,41 +64,6 @@ public class SignalsStaxXmlWriter extends StaxXmlWriter<Signals> {
writerWrapper.writeCollection("p", persons, this::writePerson);
}
public void writeWithStax(OutputStream outputStream, Signals signals, boolean useStandalone) {
try (BufferedOutputStream bos = new BufferedOutputStream(outputStream, 4 * 1024 * 1024)) {
XMLOutputFactory output = XMLOutputFactory.newInstance();
XMLEventFactory xmlEventFactory = XMLEventFactory.newInstance();
XMLEventWriter writer = output.createXMLEventWriter(bos, charsetName);
writer = new IndentingXMLEventWriter(writer);
writerWrapper = new WriterWrapper(xmlEventFactory, writer);
if (useStandalone) {
bos.write("<?xml version=\"1.0\" encoding=\"%s\" standalone=\"yes\"?>".formatted(charsetName).getBytes(charsetName));
bos.flush();
} else {
writerWrapper.addStartDocument(charsetName);
}
writerWrapper.addStartElement("Signals");
writerWrapper.addAttribute("online", signals.getOnline());
writerWrapper.addSimplelement("Id", signals.getId());
writeAuth(signals.getAuth());
writeConditions(signals.getConditions());
writerWrapper.addEndElement("Signals");
if (!useStandalone) {
writerWrapper.addEndDocument();
}
writerWrapper.flush();
System.gc();
} catch (Throwable e) {
log.error("Error serializing object to XML string: {}", e.getMessage(), e);
throw new IllegalStateException("cannot serialize object to XML");
}
}
private void writeAuth(Auth auth) throws Exception {
writerWrapper.addStartElement("Auth");
@ -106,18 +71,6 @@ public class SignalsStaxXmlWriter extends StaxXmlWriter<Signals> {
writerWrapper.addEndElement("Auth");
}
private void writeConditions(Conditions conditions) throws Exception {
writerWrapper.addStartElement("Conditions");
writePersons(conditions.getPersons());
writerWrapper.addEndElement("Conditions");
}
private void writePersons(Persons persons) throws Exception {
writerWrapper.addStartElement("Persons");
writerWrapper.writeCollection("p", persons.getPerson(), this::writePerson);
writerWrapper.addEndElement("Persons");
}
private void writePerson(Person person, String name) throws Exception {
if (person == null) {
return;
@ -158,7 +111,4 @@ public class SignalsStaxXmlWriter extends StaxXmlWriter<Signals> {
}
public interface LoaderWriter {
void accept(Consumer<List<Person>> writer);
}
}

View file

@ -1,8 +1,9 @@
package ru.nbch.credit_tracker.service.xml.stax;
import java.io.OutputStream;
import java.util.List;
import java.util.function.Consumer;
public abstract class StaxXmlWriter<T> {
public abstract class StaxXmlWriter<T, Z> {
protected final String charsetName;
protected WriterWrapper writerWrapper;
@ -11,6 +12,9 @@ public abstract class StaxXmlWriter<T> {
this.charsetName = charsetName;
}
public abstract void writeWithStax(OutputStream outputStream, T xmlObj, boolean useStandalone);
public abstract void writeWithStax(String filePath, T obj, LoaderWriter<Z> loader);
public interface LoaderWriter<Z> {
void accept(Consumer<List<Z>> writer);
}
}