diff --git a/src/main/java/ru/nbch/credit_tracker/config/settings/PipelineSettings.java b/src/main/java/ru/nbch/credit_tracker/config/settings/PipelineSettings.java index f2279df..17dcbb8 100644 --- a/src/main/java/ru/nbch/credit_tracker/config/settings/PipelineSettings.java +++ b/src/main/java/ru/nbch/credit_tracker/config/settings/PipelineSettings.java @@ -6,6 +6,7 @@ import org.springframework.boot.context.properties.bind.DefaultValue; public record PipelineSettings( @DefaultValue("5000") Integer subjectBatchSize, + @DefaultValue("3") Integer saveSubjectStageConcurrency, @DefaultValue("15") Integer nameEnricherStageConcurrency, @DefaultValue("10") Integer idEnricherStageConcurrency, @DefaultValue("5") Integer errorAttemptCount, diff --git a/src/main/java/ru/nbch/credit_tracker/service/task/PipelinePackageManager.java b/src/main/java/ru/nbch/credit_tracker/service/task/PipelinePackageManager.java index 880111e..0d480aa 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/task/PipelinePackageManager.java +++ b/src/main/java/ru/nbch/credit_tracker/service/task/PipelinePackageManager.java @@ -79,6 +79,7 @@ public class PipelinePackageManager { this.fidNameEnricher = fidNameEnricher; this.creditTrackerSettings = creditTrackerSettings; this.semaphore = new Semaphore(creditTrackerSettings.getPipeline().parallelRequests()); + log.info("pipeline settings: {}", creditTrackerSettings.getPipeline()); this.pipelineBackoff = pipelineBackoff; } @@ -87,6 +88,7 @@ public class PipelinePackageManager { throw new RuntimeException("Too many concurrent requests"); } log.info("throttler permits after acquire: {}", semaphore.availablePermits()); + long startTime = System.currentTimeMillis(); try { Pipeline pipeline = PipeLineBuilderV2.first(saveSubjectQueueFactory.getObject(), idEnricherQueueFactory.getObject()) .handler(saveSubjectHandler) @@ -110,6 +112,8 @@ public class PipelinePackageManager { .orTimeout(creditTrackerSettings.getPipeline().globalTimeout().toMillis(), TimeUnit.MILLISECONDS) .whenComplete((v, e) -> { try { + long finishTime = System.currentTimeMillis(); + log.info("pipeline time passed: {} ms", finishTime - startTime); if (e instanceof XmlMergeCoordinator.PipelineTerminatorException) { log.info("when complete: terminator was interrupted."); return; diff --git a/src/main/java/ru/nbch/credit_tracker/service/task/Stage.java b/src/main/java/ru/nbch/credit_tracker/service/task/Stage.java index 825e378..b78e508 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/task/Stage.java +++ b/src/main/java/ru/nbch/credit_tracker/service/task/Stage.java @@ -5,7 +5,7 @@ import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -13,8 +13,8 @@ import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.exception.ExceptionUtils; import ru.nbch.credit_tracker.model.StagePayload; -import ru.nbch.credit_tracker.utils.TaskTimer; import ru.nbch.credit_tracker.utils.LogUtil; +import ru.nbch.credit_tracker.utils.TaskTimer; @Slf4j public class Stage implements Runnable { @@ -42,10 +42,11 @@ public class Stage implements Ru this.out = out; this.dispatcherName = dispatcher.getClass().getSimpleName(); var mdcMap = LogUtil.extractMdcMap(); + BlockingQueue handoffQueue = new SynchronousQueue<>(); this.ioPool = new ThreadPoolExecutor(dispatcher.preferredConcurrency(), dispatcher.preferredConcurrency(), 0L, TimeUnit.MINUTES, - new LinkedBlockingQueue<>(1), + handoffQueue, r -> { Thread t = new Thread(LogUtil.wrap(mdcMap, r)); t.setName("pipe-handler-%s-%d".formatted(dispatcherName, t.threadId())); @@ -53,7 +54,13 @@ public class Stage implements Ru }, (r, executor) -> { log.info("stage {}: all threads are working; switch to CallerRunsPolicy", dispatcherName); - callerRunsPolicy.rejectedExecution(r, executor); + try { + //блокировать отправителя, пока какой‑то рабочий поток не возьмёт задачу + handoffQueue.put(r); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } +// callerRunsPolicy.rejectedExecution(r, executor); } ); } @@ -79,7 +86,6 @@ public class Stage implements Ru } f.get(); } - log.trace("Stage time stats: {}", stopWatch.info()); dispatcher.handleOnPoison(b); putQuiet(out, dispatcher.cookPoison()); log.info("stage {} finished.", dispatcherName); @@ -97,7 +103,7 @@ public class Stage implements Ru } log.info("stage {} task {} started.", dispatcher.getClass().getSimpleName(), finalCounter); Z p; - try (var m = stopWatch.start(dispatcherName)){ + try (var m = stopWatch.start(dispatcherName + "-handler")){ p = dispatcher.handle(b); } if (Thread.currentThread().isInterrupted()) { @@ -159,6 +165,7 @@ public class Stage implements Ru public void close() { log.info("Stage {} is getting shutdown", dispatcher.getClass().getSimpleName()); + log.trace("Stage time stats: {}", stopWatch.info()); ioPool.shutdownNow(); // прерываем обработчики } diff --git a/src/main/java/ru/nbch/credit_tracker/service/task/impl/SaveSubjectHandler.java b/src/main/java/ru/nbch/credit_tracker/service/task/impl/SaveSubjectHandler.java index 476c3d0..f579ca0 100644 --- a/src/main/java/ru/nbch/credit_tracker/service/task/impl/SaveSubjectHandler.java +++ b/src/main/java/ru/nbch/credit_tracker/service/task/impl/SaveSubjectHandler.java @@ -57,7 +57,7 @@ public class SaveSubjectHandler implements Handler { @@ -29,6 +30,7 @@ public final class XmlMergeCoordinator implements IPipeLineTerminator inQueue; + private final TaskTimer stopWatch = new TaskTimer(); public XmlMergeCoordinator(CreditTrackerSettings config) { this.config = config; @@ -45,9 +47,12 @@ public final class XmlMergeCoordinator implements IPipeLineTerminator