From 4eab19e87eb9dbda0d81c0d4342ce76a59942542 Mon Sep 17 00:00:00 2001 From: Ivan Nikolaev-Axenov Date: Thu, 28 May 2026 18:25:55 +0300 Subject: [PATCH] async jobs added --- .../controller/IndicSearchController.java | 8 +- .../dto/response/AddSearchJobResponseDto.java | 11 +- .../exporter/ExcelExporter.java | 6 +- .../exporter/JsonExporter.java | 6 +- .../manager/IndicSearchJobManager.java | 58 +++++ .../indicsearchservice/model/SearchJob.java | 134 +++++++++++ .../model/enums/ExportType.java | 6 + .../processor/IndicSearchProcessor.java | 216 ++++++++++++++++++ .../service/IndicSearchService.java | 143 ++---------- ...IndicSearchProcessorApplicationTests.java} | 2 +- 10 files changed, 448 insertions(+), 142 deletions(-) create mode 100644 src/main/java/ru/nbch/indicsearchservice/manager/IndicSearchJobManager.java create mode 100644 src/main/java/ru/nbch/indicsearchservice/model/SearchJob.java create mode 100644 src/main/java/ru/nbch/indicsearchservice/model/enums/ExportType.java create mode 100644 src/main/java/ru/nbch/indicsearchservice/processor/IndicSearchProcessor.java rename src/test/java/ru/nbch/indicsearchservice/{IndicSearchServiceApplicationTests.java => IndicSearchProcessorApplicationTests.java} (81%) diff --git a/src/main/java/ru/nbch/indicsearchservice/controller/IndicSearchController.java b/src/main/java/ru/nbch/indicsearchservice/controller/IndicSearchController.java index 27a57e7..7f6eb5d 100644 --- a/src/main/java/ru/nbch/indicsearchservice/controller/IndicSearchController.java +++ b/src/main/java/ru/nbch/indicsearchservice/controller/IndicSearchController.java @@ -7,6 +7,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import jakarta.validation.Valid; import java.util.Optional; +import java.util.UUID; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.validation.FieldError; @@ -24,6 +25,7 @@ import ru.nbch.indicsearchservice.dto.request.AddSearchJobRequestDto; import ru.nbch.indicsearchservice.dto.response.AddSearchJobResponseDto; import ru.nbch.indicsearchservice.dto.response.JobStatusResponseDto; import ru.nbch.indicsearchservice.exception.InternalException; +import ru.nbch.indicsearchservice.model.enums.ExportType; import ru.nbch.indicsearchservice.service.IndicSearchService; import ru.nbch.indicsearchservice.service.SettingsService; @@ -63,7 +65,7 @@ public class IndicSearchController { description = "Success") }) @GetMapping("/job_status/{id}") - public JobStatusResponseDto getJobStatus(@PathVariable("id") Long id) { + public JobStatusResponseDto getJobStatus(@PathVariable("id") UUID id) { return indicSearchService.getJobStatus(id); } @@ -77,8 +79,8 @@ public class IndicSearchController { description = "Success") }) @GetMapping("/export/{id}") - public ResponseEntity export(@PathVariable("id") Long id, - @RequestParam(name = "type", defaultValue = "JSON") String exportType) { + public ResponseEntity export(@PathVariable("id") UUID id, + @RequestParam(name = "type", defaultValue = "JSON") ExportType exportType) { return indicSearchService.export(id, exportType); } diff --git a/src/main/java/ru/nbch/indicsearchservice/dto/response/AddSearchJobResponseDto.java b/src/main/java/ru/nbch/indicsearchservice/dto/response/AddSearchJobResponseDto.java index 37ae072..1d1372b 100644 --- a/src/main/java/ru/nbch/indicsearchservice/dto/response/AddSearchJobResponseDto.java +++ b/src/main/java/ru/nbch/indicsearchservice/dto/response/AddSearchJobResponseDto.java @@ -3,25 +3,26 @@ package ru.nbch.indicsearchservice.dto.response; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.constraints.NotNull; import java.util.Objects; +import java.util.UUID; @Schema(description = "Add search job response data transfer object") public class AddSearchJobResponseDto { - @Schema(description = "Job id", example = "1") + @Schema(description = "Job id", example = "63e6dcb5-c5bd-4ce6-adb1-947ef2a2f22c") @NotNull - private Long jobId; + private UUID jobId; public AddSearchJobResponseDto() { } - public AddSearchJobResponseDto(Long jobId) { + public AddSearchJobResponseDto(UUID jobId) { this.jobId = jobId; } - public Long getJobId() { + public UUID getJobId() { return jobId; } - public void setJobId(Long jobId) { + public void setJobId(UUID jobId) { this.jobId = jobId; } diff --git a/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java b/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java index a701383..fde1475 100644 --- a/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java +++ b/src/main/java/ru/nbch/indicsearchservice/exporter/ExcelExporter.java @@ -46,9 +46,9 @@ public class ExcelExporter implements Exporter { this.settings = settings; } - public void writeExcelFile(@Nullable Path aggregatedFile, UUID taskId, Long totalRows, Map tableMetaDataMap) { + public Path writeExcelFile(@Nullable Path aggregatedFile, UUID taskId, Long totalRows, Map tableMetaDataMap) { if (aggregatedFile == null) { - return; + return null; } Path excelFile = settings.getStorageDirectory().toPath().resolve(taskId.toString() + ".xlsx"); @@ -93,6 +93,8 @@ public class ExcelExporter implements Exporter { } catch (IOException e) { log.error(e.getMessage(), e); } + + return excelFile; } private void writeToExcel(Workbook workbook, diff --git a/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java b/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java index af66908..950a97f 100644 --- a/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java +++ b/src/main/java/ru/nbch/indicsearchservice/exporter/JsonExporter.java @@ -38,12 +38,12 @@ public class JsonExporter implements Exporter { this.jsonMapper = jsonMapper; } - public void writeJsonFile(@Nullable Path aggregatedFile, + public Path writeJsonFile(@Nullable Path aggregatedFile, UUID taskId, SearchParameter searchParameter, Map tableMetaDataMap) { if (aggregatedFile == null) { - return; + return null; } boolean isPassportOgrnFid = searchParameter == SearchParameter.PASSPORT || @@ -122,6 +122,8 @@ public class JsonExporter implements Exporter { } catch (IOException e) { log.error(e.getMessage(), e); } + + return jsonFile; } private TableData buildTableDataHierarchy(List rows, Map tableMetaDataMap) { diff --git a/src/main/java/ru/nbch/indicsearchservice/manager/IndicSearchJobManager.java b/src/main/java/ru/nbch/indicsearchservice/manager/IndicSearchJobManager.java new file mode 100644 index 0000000..70b304f --- /dev/null +++ b/src/main/java/ru/nbch/indicsearchservice/manager/IndicSearchJobManager.java @@ -0,0 +1,58 @@ +package ru.nbch.indicsearchservice.manager; + +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import ru.nbch.indicsearchservice.dto.request.AddSearchJobRequestDto; +import ru.nbch.indicsearchservice.model.SearchJob; +import ru.nbch.indicsearchservice.model.enums.ExportType; +import ru.nbch.indicsearchservice.model.enums.JobStatus; +import ru.nbch.indicsearchservice.processor.IndicSearchProcessor; + +@Component +public class IndicSearchJobManager { + private final ExecutorService executor = Executors.newFixedThreadPool(5); + private final Map searchJobs = new ConcurrentHashMap<>(); + private final IndicSearchProcessor indicSearchProcessor; + + public IndicSearchJobManager(IndicSearchProcessor indicSearchProcessor) { + this.indicSearchProcessor = indicSearchProcessor; + } + + public UUID addSearchJob(AddSearchJobRequestDto request) { + UUID uuid = UUID.randomUUID(); + + SearchJob newSearchJob = new SearchJob( + uuid, + request.getDataType(), + request.getSearchParameter(), + request.getSearchValue(), + request.getMemberCode() + ); + searchJobs.put(uuid, newSearchJob); + executor.submit(() -> indicSearchProcessor.aggregateTables(newSearchJob)); + + return uuid; + } + + public JobStatus getJobStatus(UUID id) { + SearchJob searchJob = searchJobs.get(id); + if (searchJob != null) { + return searchJob.getJobStatus(); + } + return null; + } + + public ResponseEntity export(UUID id, ExportType exportType) { + SearchJob searchJob = searchJobs.get(id); + if (searchJob == null) { + return ResponseEntity.notFound().build(); + } + + return indicSearchProcessor.export(searchJob, exportType); + } +} diff --git a/src/main/java/ru/nbch/indicsearchservice/model/SearchJob.java b/src/main/java/ru/nbch/indicsearchservice/model/SearchJob.java new file mode 100644 index 0000000..51f2d46 --- /dev/null +++ b/src/main/java/ru/nbch/indicsearchservice/model/SearchJob.java @@ -0,0 +1,134 @@ +package ru.nbch.indicsearchservice.model; + +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import ru.nbch.indicsearchservice.model.enums.DataType; +import ru.nbch.indicsearchservice.model.enums.JobStatus; +import ru.nbch.indicsearchservice.model.enums.SearchParameter; + +public class SearchJob { + private final UUID uuid; + private final DataType dataType; + private final SearchParameter searchParameter; + private final String searchValue; + private final String memberCode; + + private JobStatus jobStatus; + private Path aggregatedFilePath; + private Long rowCount; + private Map tableMetaDataMap; + + private LocalDateTime createdAt; + private LocalDateTime finishedAt; + + public SearchJob(UUID uuid, DataType dataType, SearchParameter searchParameter, String searchValue, String memberCode) { + this.uuid = uuid; + this.dataType = dataType; + this.searchParameter = searchParameter; + this.searchValue = searchValue; + this.memberCode = memberCode; + + this.setJobStatus(JobStatus.QUEUED); + this.createdAt = LocalDateTime.now(); + } + + public UUID getUuid() { + return uuid; + } + + public DataType getDataType() { + return dataType; + } + + public SearchParameter getSearchParameter() { + return searchParameter; + } + + public String getSearchValue() { + return searchValue; + } + + public String getMemberCode() { + return memberCode; + } + + public JobStatus getJobStatus() { + return jobStatus; + } + + public void setJobStatus(JobStatus jobStatus) { + this.jobStatus = jobStatus; + } + + public Path getAggregatedFilePath() { + return aggregatedFilePath; + } + + public void setAggregatedFilePath(Path aggregatedFilePath) { + this.aggregatedFilePath = aggregatedFilePath; + } + + public Long getRowCount() { + return rowCount; + } + + public void setRowCount(Long rowCount) { + this.rowCount = rowCount; + } + + public Map getTableMetaDataMap() { + return tableMetaDataMap; + } + + public void setTableMetaDataMap(Map tableMetaDataMap) { + this.tableMetaDataMap = tableMetaDataMap; + } + + public LocalDateTime getCreatedAt() { + return createdAt; + } + + public void setCreatedAt(LocalDateTime createdAt) { + this.createdAt = createdAt; + } + + public LocalDateTime getFinishedAt() { + return finishedAt; + } + + public void setFinishedAt(LocalDateTime finishedAt) { + this.finishedAt = finishedAt; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + SearchJob searchJob = (SearchJob) o; + return Objects.equals(uuid, searchJob.uuid) && dataType == searchJob.dataType && searchParameter == searchJob.searchParameter && Objects.equals(searchValue, searchJob.searchValue) && Objects.equals(memberCode, searchJob.memberCode) && jobStatus == searchJob.jobStatus && Objects.equals(aggregatedFilePath, searchJob.aggregatedFilePath) && Objects.equals(rowCount, searchJob.rowCount) && Objects.equals(tableMetaDataMap, searchJob.tableMetaDataMap) && Objects.equals(createdAt, searchJob.createdAt) && Objects.equals(finishedAt, searchJob.finishedAt); + } + + @Override + public int hashCode() { + return Objects.hash(uuid, dataType, searchParameter, searchValue, memberCode, jobStatus, aggregatedFilePath, rowCount, tableMetaDataMap, createdAt, finishedAt); + } + + @Override + public String toString() { + return "SearchJob{" + + "uuid=" + uuid + + ", dataType=" + dataType + + ", searchParameter=" + searchParameter + + ", searchValue='" + searchValue + '\'' + + ", memberCode='" + memberCode + '\'' + + ", jobStatus=" + jobStatus + + ", filePath=" + aggregatedFilePath + + ", rowCount=" + rowCount + + ", tableMetaDataMap=" + tableMetaDataMap + + ", createdAt=" + createdAt + + ", finishedAt=" + finishedAt + + '}'; + } +} diff --git a/src/main/java/ru/nbch/indicsearchservice/model/enums/ExportType.java b/src/main/java/ru/nbch/indicsearchservice/model/enums/ExportType.java new file mode 100644 index 0000000..71f914d --- /dev/null +++ b/src/main/java/ru/nbch/indicsearchservice/model/enums/ExportType.java @@ -0,0 +1,6 @@ +package ru.nbch.indicsearchservice.model.enums; + +public enum ExportType { + XLSX, + JSON +} diff --git a/src/main/java/ru/nbch/indicsearchservice/processor/IndicSearchProcessor.java b/src/main/java/ru/nbch/indicsearchservice/processor/IndicSearchProcessor.java new file mode 100644 index 0000000..9d0025f --- /dev/null +++ b/src/main/java/ru/nbch/indicsearchservice/processor/IndicSearchProcessor.java @@ -0,0 +1,216 @@ +package ru.nbch.indicsearchservice.processor; + +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.lang3.mutable.MutableLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.InputStreamResource; +import org.springframework.http.ContentDisposition; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.util.StopWatch; +import ru.nbch.indicsearchservice.aggregator.FileAggregator; +import ru.nbch.indicsearchservice.exception.InternalException; +import ru.nbch.indicsearchservice.exporter.ExcelExporter; +import ru.nbch.indicsearchservice.exporter.JsonExporter; +import ru.nbch.indicsearchservice.loader.FirstLevelTableLoader; +import ru.nbch.indicsearchservice.model.SearchJob; +import ru.nbch.indicsearchservice.model.TableMetaData; +import ru.nbch.indicsearchservice.model.enums.ExportType; +import ru.nbch.indicsearchservice.model.enums.JobStatus; +import ru.nbch.indicsearchservice.model.enums.SearchParameter; +import ru.nbch.indicsearchservice.util.TableMetaDataUtil; + +@Component +public class IndicSearchProcessor { + private static final Logger log = LoggerFactory.getLogger(IndicSearchProcessor.class); + + private final List tableLoaders; + private final List fileAggregators; + private final ExcelExporter excelExporter; + private final JsonExporter jsonExporter; + private final TableMetaDataUtil tableMetaDataUtil; + + public IndicSearchProcessor(List tableLoaders, + List fileAggregators, + ExcelExporter excelExporter, + JsonExporter jsonExporter, + TableMetaDataUtil tableMetaDataUtil) { + this.tableLoaders = tableLoaders; + this.fileAggregators = fileAggregators; + this.excelExporter = excelExporter; + this.jsonExporter = jsonExporter; + this.tableMetaDataUtil = tableMetaDataUtil; + } + + public void aggregateTables(SearchJob job) { + job.setJobStatus(JobStatus.IN_PROGRESS); + + StopWatch sw = new StopWatch(); + + // Подготовительный этап. Получение мета данных из БД + log.info("Prep step started. Getting metadata from database"); + sw.start(); + + Map tableMetaDataMap = tableMetaDataUtil.prepareTableMetaData(job.getDataType(), job.getSearchParameter()); + job.setTableMetaDataMap(tableMetaDataMap); + + sw.stop(); + log.info(sw.prettyPrint()); + sw = new StopWatch(); + + // Первый этап. Получение данных из БД и заполнение файлов + log.info("First step started. Getting data from database and filling csv files"); + sw.start(); + + Map tableFileMap = new ConcurrentHashMap<>(); + tableLoaders.parallelStream().forEach(tl -> tl.searchBy( + job.getUuid(), + job.getDataType(), + job.getSearchParameter(), + job.getSearchValue(), + job.getMemberCode().substring(0, 6), + tableFileMap, + tableMetaDataMap + )); + + sw.stop(); + log.info(sw.prettyPrint()); + sw = new StopWatch(); + + // Второй этап. Объединение файлов в один + log.info("Second step started. Aggregating table files into one."); + sw.start(); + + MutableLong rowCounter = new MutableLong(); + Path aggregatedFile = fileAggregators.stream().map(fa -> fa.aggregateFiles( + job.getUuid(), + job.getDataType(), + job.getSearchParameter(), + job.getSearchValue(), + job.getMemberCode().substring(0, 6), + tableFileMap, + rowCounter)) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + + job.setAggregatedFilePath(aggregatedFile); + job.setRowCount(rowCounter.longValue()); + job.setFinishedAt(LocalDateTime.now()); + job.setJobStatus(JobStatus.DONE); + + sw.stop(); + log.info(sw.prettyPrint()); + + for (Path tableFilePath : tableFileMap.values()) { + try { + Files.deleteIfExists(tableFilePath); + } catch (IOException e) { + throw new InternalException("Failed to delete file " + tableFilePath + " " + e); + } + } + } + + public ResponseEntity export(SearchJob job, ExportType exportType) { + StopWatch sw = new StopWatch(); + sw.start(); + + UUID taskId = job.getUuid(); + SearchParameter searchParameter = job.getSearchParameter(); + + Path aggregatedFile = job.getAggregatedFilePath(); + + long rowCount = job.getRowCount(); + Map tableMetaDataMap = job.getTableMetaDataMap(); + + Path exportedFile; + String fileName = "err.txt"; + if (exportType == ExportType.JSON) { + fileName = "output.json"; + exportedFile = jsonExporter.writeJsonFile(aggregatedFile, taskId, searchParameter, tableMetaDataMap); + } else if (exportType == ExportType.XLSX) { + fileName = "NBKI table.xlsx"; + tableMetaDataUtil.fillTableMetaDataWithLocalization(tableMetaDataMap); + exportedFile = excelExporter.writeExcelFile(aggregatedFile, taskId, rowCount, tableMetaDataMap); + } else { + exportedFile = null; + } + +// long rowLimit = Long.parseLong(settingsService.getSettingsValueByName("LimitDB")); +// if (rowCount < rowLimit && !IS_EXPORT_AS_EXCEL) { +// jsonExporter.writeJsonFile(aggregatedFile, taskId, searchParameter, tableMetaDataMap); +// } else { +// tableMetaDataUtil.fillTableMetaDataWithLocalization(tableMetaDataMap); +// excelExporter.writeExcelFile(aggregatedFile, taskId, rowCount, tableMetaDataMap); +// } + + sw.stop(); + log.info(sw.prettyPrint()); + + if (exportedFile == null) { + return ResponseEntity.internalServerError().build(); + } + + long contentLength; + try { + contentLength = Files.size(exportedFile); + } catch (IOException e) { + log.error("Failed to get temp excel file size!", e); + try { + if (aggregatedFile != null) { + Files.deleteIfExists(aggregatedFile); + } + Files.deleteIfExists(exportedFile); + } catch (IOException ignored) { + } + throw new InternalException("Failed to get exported file size! " + e); + } + + InputStreamResource exportedFileResource; + try { + exportedFileResource = new InputStreamResource(new FileInputStream(exportedFile.toFile()) { + @Override + public void close() throws IOException { + super.close(); + if (aggregatedFile != null) { + Files.deleteIfExists(aggregatedFile); + } + Files.deleteIfExists(exportedFile); + } + }); + } catch (IOException e) { + log.error("Failed to create exported file resource!", e); + try { + if (aggregatedFile != null) { + Files.deleteIfExists(aggregatedFile); + } + Files.deleteIfExists(exportedFile); + } catch (IOException ignored) { + } + throw new InternalException("Failed to create exported file resource! " + e); + } + + return ResponseEntity.ok() + .contentType(MediaType.APPLICATION_OCTET_STREAM) + .contentLength(contentLength) + .header(HttpHeaders.CONTENT_DISPOSITION, + ContentDisposition.attachment() + .filename(fileName, StandardCharsets.UTF_8) + .build() + .toString()) + .body(exportedFileResource); + } +} diff --git a/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java b/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java index acb814c..76c69ab 100644 --- a/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java +++ b/src/main/java/ru/nbch/indicsearchservice/service/IndicSearchService.java @@ -1,151 +1,36 @@ package ru.nbch.indicsearchservice.service; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.commons.lang3.mutable.MutableLong; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; -import org.springframework.util.StopWatch; -import ru.nbch.indicsearchservice.aggregator.FileAggregator; import ru.nbch.indicsearchservice.dto.request.AddSearchJobRequestDto; import ru.nbch.indicsearchservice.dto.response.AddSearchJobResponseDto; import ru.nbch.indicsearchservice.dto.response.JobStatusResponseDto; -import ru.nbch.indicsearchservice.exception.InternalException; -import ru.nbch.indicsearchservice.exporter.ExcelExporter; -import ru.nbch.indicsearchservice.exporter.JsonExporter; -import ru.nbch.indicsearchservice.loader.FirstLevelTableLoader; -import ru.nbch.indicsearchservice.model.TableMetaData; -import ru.nbch.indicsearchservice.util.TableMetaDataUtil; +import ru.nbch.indicsearchservice.manager.IndicSearchJobManager; +import ru.nbch.indicsearchservice.model.enums.ExportType; +import ru.nbch.indicsearchservice.model.enums.JobStatus; @Service public class IndicSearchService { - private static final Logger log = LoggerFactory.getLogger(IndicSearchService.class); + private final IndicSearchJobManager indicSearchJobManager; - private static final boolean IS_EXPORT_AS_EXCEL = false; - - private final List tableLoaders; - private final List fileAggregators; - private final ExcelExporter excelExporter; - private final JsonExporter jsonExporter; - private final SettingsService settingsService; - private final TableMetaDataUtil tableMetaDataUtil; - - public IndicSearchService(List tableLoaders, - List fileAggregators, - ExcelExporter excelExporter, - JsonExporter jsonExporter, - SettingsService settingsService, - TableMetaDataUtil tableMetaDataUtil) { - this.tableLoaders = tableLoaders; - this.fileAggregators = fileAggregators; - this.excelExporter = excelExporter; - this.jsonExporter = jsonExporter; - this.settingsService = settingsService; - this.tableMetaDataUtil = tableMetaDataUtil; + public IndicSearchService(IndicSearchJobManager indicSearchJobManager) { + this.indicSearchJobManager = indicSearchJobManager; } public AddSearchJobResponseDto addSearchJob(AddSearchJobRequestDto request) { - UUID taskId = UUID.randomUUID(); + UUID jobId = indicSearchJobManager.addSearchJob(request); - StopWatch sw = new StopWatch(); - - // Подготовительный этап. Получение мета данных из БД - log.info("Prep step started. Getting metadata from database"); - sw.start(); - - Map tableMetaDataMap = tableMetaDataUtil.prepareTableMetaData(request.getDataType(), request.getSearchParameter()); - long rowLimit = Long.parseLong(settingsService.getSettingsValueByName("LimitDB")); - - sw.stop(); - log.info(sw.prettyPrint()); - sw = new StopWatch(); - - // Первый этап. Получение данных из БД и заполнение файлов - log.info("First step started. Getting data from database and filling csv files"); - sw.start(); - - Map tableFileMap = new ConcurrentHashMap<>(); - tableLoaders.parallelStream().forEach(tl -> tl.searchBy( - taskId, - request.getDataType(), - request.getSearchParameter(), - request.getSearchValue(), - request.getMemberCode().substring(0, 6), - tableFileMap, - tableMetaDataMap - )); - - sw.stop(); - log.info(sw.prettyPrint()); - sw = new StopWatch(); - - // Второй этап. Объединение файлов в один - log.info("Second step started. Aggregating table files into one."); - sw.start(); - - MutableLong rowCounter = new MutableLong(); - Path aggregatedFile = fileAggregators.stream().map(fa -> fa.aggregateFiles( - taskId, - request.getDataType(), - request.getSearchParameter(), - request.getSearchValue(), - request.getMemberCode().substring(0, 6), - tableFileMap, - rowCounter)) - .filter(Objects::nonNull) - .findFirst() - .orElse(null); - - sw.stop(); - log.info(sw.prettyPrint()); - sw = new StopWatch(); - - for (Path tableFilePath : tableFileMap.values()) { - try { - Files.deleteIfExists(tableFilePath); - } catch (IOException e) { - throw new InternalException("Failed to delete file " + tableFilePath + " " + e); - } - } - - // Третий этап. Экспорт агрегированного файла в JSON или XLSX - log.info("Third step started. Exporting aggregated file into JSON/XLSX."); - sw.start(); - - if (rowCounter.longValue() < rowLimit && !IS_EXPORT_AS_EXCEL) { - jsonExporter.writeJsonFile(aggregatedFile, taskId, request.getSearchParameter(), tableMetaDataMap); - } else { - tableMetaDataUtil.fillTableMetaDataWithLocalization(tableMetaDataMap); - excelExporter.writeExcelFile(aggregatedFile, taskId, rowCounter.longValue(), tableMetaDataMap); - } - - sw.stop(); - log.info(sw.prettyPrint()); - - try { - if (aggregatedFile != null) { - Files.deleteIfExists(aggregatedFile); - } - } catch (IOException e) { - throw new InternalException("Failed to delete file " + aggregatedFile + " " + e); - } - - return null; + return new AddSearchJobResponseDto(jobId); } - public JobStatusResponseDto getJobStatus(Long id) { - return null; + public JobStatusResponseDto getJobStatus(UUID id) { + JobStatus jobStatus = indicSearchJobManager.getJobStatus(id); + + return new JobStatusResponseDto(jobStatus); } - public ResponseEntity export(Long id, String exportType) { - return null; + public ResponseEntity export(UUID id, ExportType exportType) { + return indicSearchJobManager.export(id, exportType); } } diff --git a/src/test/java/ru/nbch/indicsearchservice/IndicSearchServiceApplicationTests.java b/src/test/java/ru/nbch/indicsearchservice/IndicSearchProcessorApplicationTests.java similarity index 81% rename from src/test/java/ru/nbch/indicsearchservice/IndicSearchServiceApplicationTests.java rename to src/test/java/ru/nbch/indicsearchservice/IndicSearchProcessorApplicationTests.java index d974fc7..220bbc1 100644 --- a/src/test/java/ru/nbch/indicsearchservice/IndicSearchServiceApplicationTests.java +++ b/src/test/java/ru/nbch/indicsearchservice/IndicSearchProcessorApplicationTests.java @@ -4,7 +4,7 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; @SpringBootTest -class IndicSearchServiceApplicationTests { +class IndicSearchProcessorApplicationTests { @Test void contextLoads() {