async jobs added

This commit is contained in:
Ivan Nikolaev-Axenov 2026-05-28 18:25:55 +03:00
parent cb2b3dddef
commit 4eab19e87e
10 changed files with 448 additions and 142 deletions

View file

@ -7,6 +7,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import jakarta.validation.Valid;
import java.util.Optional;
import java.util.UUID;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.validation.FieldError;
@ -24,6 +25,7 @@ import ru.nbch.indicsearchservice.dto.request.AddSearchJobRequestDto;
import ru.nbch.indicsearchservice.dto.response.AddSearchJobResponseDto;
import ru.nbch.indicsearchservice.dto.response.JobStatusResponseDto;
import ru.nbch.indicsearchservice.exception.InternalException;
import ru.nbch.indicsearchservice.model.enums.ExportType;
import ru.nbch.indicsearchservice.service.IndicSearchService;
import ru.nbch.indicsearchservice.service.SettingsService;
@ -63,7 +65,7 @@ public class IndicSearchController {
description = "Success")
})
@GetMapping("/job_status/{id}")
public JobStatusResponseDto getJobStatus(@PathVariable("id") Long id) {
public JobStatusResponseDto getJobStatus(@PathVariable("id") UUID id) {
return indicSearchService.getJobStatus(id);
}
@ -77,8 +79,8 @@ public class IndicSearchController {
description = "Success")
})
@GetMapping("/export/{id}")
public ResponseEntity<?> export(@PathVariable("id") Long id,
@RequestParam(name = "type", defaultValue = "JSON") String exportType) {
public ResponseEntity<?> export(@PathVariable("id") UUID id,
@RequestParam(name = "type", defaultValue = "JSON") ExportType exportType) {
return indicSearchService.export(id, exportType);
}

View file

@ -3,25 +3,26 @@ package ru.nbch.indicsearchservice.dto.response;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import java.util.Objects;
import java.util.UUID;
@Schema(description = "Add search job response data transfer object")
public class AddSearchJobResponseDto {
@Schema(description = "Job id", example = "1")
@Schema(description = "Job id", example = "63e6dcb5-c5bd-4ce6-adb1-947ef2a2f22c")
@NotNull
private Long jobId;
private UUID jobId;
public AddSearchJobResponseDto() {
}
public AddSearchJobResponseDto(Long jobId) {
public AddSearchJobResponseDto(UUID jobId) {
this.jobId = jobId;
}
public Long getJobId() {
public UUID getJobId() {
return jobId;
}
public void setJobId(Long jobId) {
public void setJobId(UUID jobId) {
this.jobId = jobId;
}

View file

@ -46,9 +46,9 @@ public class ExcelExporter implements Exporter {
this.settings = settings;
}
public void writeExcelFile(@Nullable Path aggregatedFile, UUID taskId, Long totalRows, Map<String, TableMetaData> tableMetaDataMap) {
public Path writeExcelFile(@Nullable Path aggregatedFile, UUID taskId, Long totalRows, Map<String, TableMetaData> tableMetaDataMap) {
if (aggregatedFile == null) {
return;
return null;
}
Path excelFile = settings.getStorageDirectory().toPath().resolve(taskId.toString() + ".xlsx");
@ -93,6 +93,8 @@ public class ExcelExporter implements Exporter {
} catch (IOException e) {
log.error(e.getMessage(), e);
}
return excelFile;
}
private void writeToExcel(Workbook workbook,

View file

@ -38,12 +38,12 @@ public class JsonExporter implements Exporter {
this.jsonMapper = jsonMapper;
}
public void writeJsonFile(@Nullable Path aggregatedFile,
public Path writeJsonFile(@Nullable Path aggregatedFile,
UUID taskId,
SearchParameter searchParameter,
Map<String, TableMetaData> tableMetaDataMap) {
if (aggregatedFile == null) {
return;
return null;
}
boolean isPassportOgrnFid = searchParameter == SearchParameter.PASSPORT ||
@ -122,6 +122,8 @@ public class JsonExporter implements Exporter {
} catch (IOException e) {
log.error(e.getMessage(), e);
}
return jsonFile;
}
private TableData buildTableDataHierarchy(List<String> rows, Map<String, TableMetaData> tableMetaDataMap) {

View file

@ -0,0 +1,58 @@
package ru.nbch.indicsearchservice.manager;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import ru.nbch.indicsearchservice.dto.request.AddSearchJobRequestDto;
import ru.nbch.indicsearchservice.model.SearchJob;
import ru.nbch.indicsearchservice.model.enums.ExportType;
import ru.nbch.indicsearchservice.model.enums.JobStatus;
import ru.nbch.indicsearchservice.processor.IndicSearchProcessor;
@Component
public class IndicSearchJobManager {
private final ExecutorService executor = Executors.newFixedThreadPool(5);
private final Map<UUID, SearchJob> searchJobs = new ConcurrentHashMap<>();
private final IndicSearchProcessor indicSearchProcessor;
public IndicSearchJobManager(IndicSearchProcessor indicSearchProcessor) {
this.indicSearchProcessor = indicSearchProcessor;
}
public UUID addSearchJob(AddSearchJobRequestDto request) {
UUID uuid = UUID.randomUUID();
SearchJob newSearchJob = new SearchJob(
uuid,
request.getDataType(),
request.getSearchParameter(),
request.getSearchValue(),
request.getMemberCode()
);
searchJobs.put(uuid, newSearchJob);
executor.submit(() -> indicSearchProcessor.aggregateTables(newSearchJob));
return uuid;
}
public JobStatus getJobStatus(UUID id) {
SearchJob searchJob = searchJobs.get(id);
if (searchJob != null) {
return searchJob.getJobStatus();
}
return null;
}
public ResponseEntity<?> export(UUID id, ExportType exportType) {
SearchJob searchJob = searchJobs.get(id);
if (searchJob == null) {
return ResponseEntity.notFound().build();
}
return indicSearchProcessor.export(searchJob, exportType);
}
}

View file

@ -0,0 +1,134 @@
package ru.nbch.indicsearchservice.model;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import ru.nbch.indicsearchservice.model.enums.DataType;
import ru.nbch.indicsearchservice.model.enums.JobStatus;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
public class SearchJob {
private final UUID uuid;
private final DataType dataType;
private final SearchParameter searchParameter;
private final String searchValue;
private final String memberCode;
private JobStatus jobStatus;
private Path aggregatedFilePath;
private Long rowCount;
private Map<String, TableMetaData> tableMetaDataMap;
private LocalDateTime createdAt;
private LocalDateTime finishedAt;
public SearchJob(UUID uuid, DataType dataType, SearchParameter searchParameter, String searchValue, String memberCode) {
this.uuid = uuid;
this.dataType = dataType;
this.searchParameter = searchParameter;
this.searchValue = searchValue;
this.memberCode = memberCode;
this.setJobStatus(JobStatus.QUEUED);
this.createdAt = LocalDateTime.now();
}
public UUID getUuid() {
return uuid;
}
public DataType getDataType() {
return dataType;
}
public SearchParameter getSearchParameter() {
return searchParameter;
}
public String getSearchValue() {
return searchValue;
}
public String getMemberCode() {
return memberCode;
}
public JobStatus getJobStatus() {
return jobStatus;
}
public void setJobStatus(JobStatus jobStatus) {
this.jobStatus = jobStatus;
}
public Path getAggregatedFilePath() {
return aggregatedFilePath;
}
public void setAggregatedFilePath(Path aggregatedFilePath) {
this.aggregatedFilePath = aggregatedFilePath;
}
public Long getRowCount() {
return rowCount;
}
public void setRowCount(Long rowCount) {
this.rowCount = rowCount;
}
public Map<String, TableMetaData> getTableMetaDataMap() {
return tableMetaDataMap;
}
public void setTableMetaDataMap(Map<String, TableMetaData> tableMetaDataMap) {
this.tableMetaDataMap = tableMetaDataMap;
}
public LocalDateTime getCreatedAt() {
return createdAt;
}
public void setCreatedAt(LocalDateTime createdAt) {
this.createdAt = createdAt;
}
public LocalDateTime getFinishedAt() {
return finishedAt;
}
public void setFinishedAt(LocalDateTime finishedAt) {
this.finishedAt = finishedAt;
}
@Override
public boolean equals(Object o) {
if (o == null || getClass() != o.getClass()) return false;
SearchJob searchJob = (SearchJob) o;
return Objects.equals(uuid, searchJob.uuid) && dataType == searchJob.dataType && searchParameter == searchJob.searchParameter && Objects.equals(searchValue, searchJob.searchValue) && Objects.equals(memberCode, searchJob.memberCode) && jobStatus == searchJob.jobStatus && Objects.equals(aggregatedFilePath, searchJob.aggregatedFilePath) && Objects.equals(rowCount, searchJob.rowCount) && Objects.equals(tableMetaDataMap, searchJob.tableMetaDataMap) && Objects.equals(createdAt, searchJob.createdAt) && Objects.equals(finishedAt, searchJob.finishedAt);
}
@Override
public int hashCode() {
return Objects.hash(uuid, dataType, searchParameter, searchValue, memberCode, jobStatus, aggregatedFilePath, rowCount, tableMetaDataMap, createdAt, finishedAt);
}
@Override
public String toString() {
return "SearchJob{" +
"uuid=" + uuid +
", dataType=" + dataType +
", searchParameter=" + searchParameter +
", searchValue='" + searchValue + '\'' +
", memberCode='" + memberCode + '\'' +
", jobStatus=" + jobStatus +
", filePath=" + aggregatedFilePath +
", rowCount=" + rowCount +
", tableMetaDataMap=" + tableMetaDataMap +
", createdAt=" + createdAt +
", finishedAt=" + finishedAt +
'}';
}
}

View file

@ -0,0 +1,6 @@
package ru.nbch.indicsearchservice.model.enums;
public enum ExportType {
XLSX,
JSON
}

View file

@ -0,0 +1,216 @@
package ru.nbch.indicsearchservice.processor;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang3.mutable.MutableLong;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.ContentDisposition;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.util.StopWatch;
import ru.nbch.indicsearchservice.aggregator.FileAggregator;
import ru.nbch.indicsearchservice.exception.InternalException;
import ru.nbch.indicsearchservice.exporter.ExcelExporter;
import ru.nbch.indicsearchservice.exporter.JsonExporter;
import ru.nbch.indicsearchservice.loader.FirstLevelTableLoader;
import ru.nbch.indicsearchservice.model.SearchJob;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.model.enums.ExportType;
import ru.nbch.indicsearchservice.model.enums.JobStatus;
import ru.nbch.indicsearchservice.model.enums.SearchParameter;
import ru.nbch.indicsearchservice.util.TableMetaDataUtil;
@Component
public class IndicSearchProcessor {
private static final Logger log = LoggerFactory.getLogger(IndicSearchProcessor.class);
private final List<FirstLevelTableLoader> tableLoaders;
private final List<FileAggregator> fileAggregators;
private final ExcelExporter excelExporter;
private final JsonExporter jsonExporter;
private final TableMetaDataUtil tableMetaDataUtil;
public IndicSearchProcessor(List<FirstLevelTableLoader> tableLoaders,
List<FileAggregator> fileAggregators,
ExcelExporter excelExporter,
JsonExporter jsonExporter,
TableMetaDataUtil tableMetaDataUtil) {
this.tableLoaders = tableLoaders;
this.fileAggregators = fileAggregators;
this.excelExporter = excelExporter;
this.jsonExporter = jsonExporter;
this.tableMetaDataUtil = tableMetaDataUtil;
}
public void aggregateTables(SearchJob job) {
job.setJobStatus(JobStatus.IN_PROGRESS);
StopWatch sw = new StopWatch();
// Подготовительный этап. Получение мета данных из БД
log.info("Prep step started. Getting metadata from database");
sw.start();
Map<String, TableMetaData> tableMetaDataMap = tableMetaDataUtil.prepareTableMetaData(job.getDataType(), job.getSearchParameter());
job.setTableMetaDataMap(tableMetaDataMap);
sw.stop();
log.info(sw.prettyPrint());
sw = new StopWatch();
// Первый этап. Получение данных из БД и заполнение файлов
log.info("First step started. Getting data from database and filling csv files");
sw.start();
Map<String, Path> tableFileMap = new ConcurrentHashMap<>();
tableLoaders.parallelStream().forEach(tl -> tl.searchBy(
job.getUuid(),
job.getDataType(),
job.getSearchParameter(),
job.getSearchValue(),
job.getMemberCode().substring(0, 6),
tableFileMap,
tableMetaDataMap
));
sw.stop();
log.info(sw.prettyPrint());
sw = new StopWatch();
// Второй этап. Объединение файлов в один
log.info("Second step started. Aggregating table files into one.");
sw.start();
MutableLong rowCounter = new MutableLong();
Path aggregatedFile = fileAggregators.stream().map(fa -> fa.aggregateFiles(
job.getUuid(),
job.getDataType(),
job.getSearchParameter(),
job.getSearchValue(),
job.getMemberCode().substring(0, 6),
tableFileMap,
rowCounter))
.filter(Objects::nonNull)
.findFirst()
.orElse(null);
job.setAggregatedFilePath(aggregatedFile);
job.setRowCount(rowCounter.longValue());
job.setFinishedAt(LocalDateTime.now());
job.setJobStatus(JobStatus.DONE);
sw.stop();
log.info(sw.prettyPrint());
for (Path tableFilePath : tableFileMap.values()) {
try {
Files.deleteIfExists(tableFilePath);
} catch (IOException e) {
throw new InternalException("Failed to delete file " + tableFilePath + " " + e);
}
}
}
public ResponseEntity<?> export(SearchJob job, ExportType exportType) {
StopWatch sw = new StopWatch();
sw.start();
UUID taskId = job.getUuid();
SearchParameter searchParameter = job.getSearchParameter();
Path aggregatedFile = job.getAggregatedFilePath();
long rowCount = job.getRowCount();
Map<String, TableMetaData> tableMetaDataMap = job.getTableMetaDataMap();
Path exportedFile;
String fileName = "err.txt";
if (exportType == ExportType.JSON) {
fileName = "output.json";
exportedFile = jsonExporter.writeJsonFile(aggregatedFile, taskId, searchParameter, tableMetaDataMap);
} else if (exportType == ExportType.XLSX) {
fileName = "NBKI table.xlsx";
tableMetaDataUtil.fillTableMetaDataWithLocalization(tableMetaDataMap);
exportedFile = excelExporter.writeExcelFile(aggregatedFile, taskId, rowCount, tableMetaDataMap);
} else {
exportedFile = null;
}
// long rowLimit = Long.parseLong(settingsService.getSettingsValueByName("LimitDB"));
// if (rowCount < rowLimit && !IS_EXPORT_AS_EXCEL) {
// jsonExporter.writeJsonFile(aggregatedFile, taskId, searchParameter, tableMetaDataMap);
// } else {
// tableMetaDataUtil.fillTableMetaDataWithLocalization(tableMetaDataMap);
// excelExporter.writeExcelFile(aggregatedFile, taskId, rowCount, tableMetaDataMap);
// }
sw.stop();
log.info(sw.prettyPrint());
if (exportedFile == null) {
return ResponseEntity.internalServerError().build();
}
long contentLength;
try {
contentLength = Files.size(exportedFile);
} catch (IOException e) {
log.error("Failed to get temp excel file size!", e);
try {
if (aggregatedFile != null) {
Files.deleteIfExists(aggregatedFile);
}
Files.deleteIfExists(exportedFile);
} catch (IOException ignored) {
}
throw new InternalException("Failed to get exported file size! " + e);
}
InputStreamResource exportedFileResource;
try {
exportedFileResource = new InputStreamResource(new FileInputStream(exportedFile.toFile()) {
@Override
public void close() throws IOException {
super.close();
if (aggregatedFile != null) {
Files.deleteIfExists(aggregatedFile);
}
Files.deleteIfExists(exportedFile);
}
});
} catch (IOException e) {
log.error("Failed to create exported file resource!", e);
try {
if (aggregatedFile != null) {
Files.deleteIfExists(aggregatedFile);
}
Files.deleteIfExists(exportedFile);
} catch (IOException ignored) {
}
throw new InternalException("Failed to create exported file resource! " + e);
}
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.contentLength(contentLength)
.header(HttpHeaders.CONTENT_DISPOSITION,
ContentDisposition.attachment()
.filename(fileName, StandardCharsets.UTF_8)
.build()
.toString())
.body(exportedFileResource);
}
}

View file

@ -1,151 +1,36 @@
package ru.nbch.indicsearchservice.service;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang3.mutable.MutableLong;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.util.StopWatch;
import ru.nbch.indicsearchservice.aggregator.FileAggregator;
import ru.nbch.indicsearchservice.dto.request.AddSearchJobRequestDto;
import ru.nbch.indicsearchservice.dto.response.AddSearchJobResponseDto;
import ru.nbch.indicsearchservice.dto.response.JobStatusResponseDto;
import ru.nbch.indicsearchservice.exception.InternalException;
import ru.nbch.indicsearchservice.exporter.ExcelExporter;
import ru.nbch.indicsearchservice.exporter.JsonExporter;
import ru.nbch.indicsearchservice.loader.FirstLevelTableLoader;
import ru.nbch.indicsearchservice.model.TableMetaData;
import ru.nbch.indicsearchservice.util.TableMetaDataUtil;
import ru.nbch.indicsearchservice.manager.IndicSearchJobManager;
import ru.nbch.indicsearchservice.model.enums.ExportType;
import ru.nbch.indicsearchservice.model.enums.JobStatus;
@Service
public class IndicSearchService {
private static final Logger log = LoggerFactory.getLogger(IndicSearchService.class);
private final IndicSearchJobManager indicSearchJobManager;
private static final boolean IS_EXPORT_AS_EXCEL = false;
private final List<FirstLevelTableLoader> tableLoaders;
private final List<FileAggregator> fileAggregators;
private final ExcelExporter excelExporter;
private final JsonExporter jsonExporter;
private final SettingsService settingsService;
private final TableMetaDataUtil tableMetaDataUtil;
public IndicSearchService(List<FirstLevelTableLoader> tableLoaders,
List<FileAggregator> fileAggregators,
ExcelExporter excelExporter,
JsonExporter jsonExporter,
SettingsService settingsService,
TableMetaDataUtil tableMetaDataUtil) {
this.tableLoaders = tableLoaders;
this.fileAggregators = fileAggregators;
this.excelExporter = excelExporter;
this.jsonExporter = jsonExporter;
this.settingsService = settingsService;
this.tableMetaDataUtil = tableMetaDataUtil;
public IndicSearchService(IndicSearchJobManager indicSearchJobManager) {
this.indicSearchJobManager = indicSearchJobManager;
}
public AddSearchJobResponseDto addSearchJob(AddSearchJobRequestDto request) {
UUID taskId = UUID.randomUUID();
UUID jobId = indicSearchJobManager.addSearchJob(request);
StopWatch sw = new StopWatch();
// Подготовительный этап. Получение мета данных из БД
log.info("Prep step started. Getting metadata from database");
sw.start();
Map<String, TableMetaData> tableMetaDataMap = tableMetaDataUtil.prepareTableMetaData(request.getDataType(), request.getSearchParameter());
long rowLimit = Long.parseLong(settingsService.getSettingsValueByName("LimitDB"));
sw.stop();
log.info(sw.prettyPrint());
sw = new StopWatch();
// Первый этап. Получение данных из БД и заполнение файлов
log.info("First step started. Getting data from database and filling csv files");
sw.start();
Map<String, Path> tableFileMap = new ConcurrentHashMap<>();
tableLoaders.parallelStream().forEach(tl -> tl.searchBy(
taskId,
request.getDataType(),
request.getSearchParameter(),
request.getSearchValue(),
request.getMemberCode().substring(0, 6),
tableFileMap,
tableMetaDataMap
));
sw.stop();
log.info(sw.prettyPrint());
sw = new StopWatch();
// Второй этап. Объединение файлов в один
log.info("Second step started. Aggregating table files into one.");
sw.start();
MutableLong rowCounter = new MutableLong();
Path aggregatedFile = fileAggregators.stream().map(fa -> fa.aggregateFiles(
taskId,
request.getDataType(),
request.getSearchParameter(),
request.getSearchValue(),
request.getMemberCode().substring(0, 6),
tableFileMap,
rowCounter))
.filter(Objects::nonNull)
.findFirst()
.orElse(null);
sw.stop();
log.info(sw.prettyPrint());
sw = new StopWatch();
for (Path tableFilePath : tableFileMap.values()) {
try {
Files.deleteIfExists(tableFilePath);
} catch (IOException e) {
throw new InternalException("Failed to delete file " + tableFilePath + " " + e);
}
return new AddSearchJobResponseDto(jobId);
}
// Третий этап. Экспорт агрегированного файла в JSON или XLSX
log.info("Third step started. Exporting aggregated file into JSON/XLSX.");
sw.start();
public JobStatusResponseDto getJobStatus(UUID id) {
JobStatus jobStatus = indicSearchJobManager.getJobStatus(id);
if (rowCounter.longValue() < rowLimit && !IS_EXPORT_AS_EXCEL) {
jsonExporter.writeJsonFile(aggregatedFile, taskId, request.getSearchParameter(), tableMetaDataMap);
} else {
tableMetaDataUtil.fillTableMetaDataWithLocalization(tableMetaDataMap);
excelExporter.writeExcelFile(aggregatedFile, taskId, rowCounter.longValue(), tableMetaDataMap);
return new JobStatusResponseDto(jobStatus);
}
sw.stop();
log.info(sw.prettyPrint());
try {
if (aggregatedFile != null) {
Files.deleteIfExists(aggregatedFile);
}
} catch (IOException e) {
throw new InternalException("Failed to delete file " + aggregatedFile + " " + e);
}
return null;
}
public JobStatusResponseDto getJobStatus(Long id) {
return null;
}
public ResponseEntity<?> export(Long id, String exportType) {
return null;
public ResponseEntity<?> export(UUID id, ExportType exportType) {
return indicSearchJobManager.export(id, exportType);
}
}

View file

@ -4,7 +4,7 @@ import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest
class IndicSearchServiceApplicationTests {
class IndicSearchProcessorApplicationTests {
@Test
void contextLoads() {