treshenkov/credit_tracker
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pipeline bugfixes

Browse source
This commit is contained in:
ialbert 2025-11-11 16:59:18 +03:00
parent 395613ae01
commit 945694e50d
2 changed files with 42 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
src/main/java/ru/nbch/credit_tracker/service/task/Stage.java
View file

@ -4,7 +4,6 @@ import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
@ -22,8 +21,11 @@ public class Stage<X extends StagePayload, Z extends StagePayload> implements Ru
private final BlockingQueue<Z> out; private final BlockingQueue<Z> out;
private final Z POISON; private final Z POISON;
private final ExecutorService ioPool; private final ExecutorService ioPool;
private final CompletableFuture<Void> done = new CompletableFuture<>();
private Consumer<Z> consumer = null; private Consumer<Z> consumer = null;
private final String dispatcherName;
private static final ThreadPoolExecutor.CallerRunsPolicy callerRunsPolicy
= new ThreadPoolExecutor.CallerRunsPolicy();
public Stage(Handler<X, Z> dispatcher, public Stage(Handler<X, Z> dispatcher,
BlockingQueue<X> in, BlockingQueue<X> in,
@ -32,24 +34,24 @@ public class Stage<X extends StagePayload, Z extends StagePayload> implements Ru
this.dispatcher = dispatcher; this.dispatcher = dispatcher;
this.in = in; this.in = in;
this.out = out; this.out = out;
this.dispatcherName = dispatcher.getClass().getSimpleName();
this.ioPool = new ThreadPoolExecutor(10, this.ioPool = new ThreadPoolExecutor(10,
10, 10,
0L, TimeUnit.MINUTES, 0L, TimeUnit.MINUTES,
new LinkedBlockingQueue<>(1), new LinkedBlockingQueue<>(1),
r -> { r -> {
Thread t = new Thread(r); Thread t = new Thread(r);
t.setName("pipe-handler-%s-%d".formatted(dispatcher.getClass().getSimpleName(), t.threadId())); t.setName("pipe-handler-%s-%d".formatted(dispatcherName, t.threadId()));
return t; return t;
}, },
new ThreadPoolExecutor.CallerRunsPolicy() (r, executor) -> {
log.info("stage {}: all threads are working; switch to CallerRunsPolicy", dispatcherName);
callerRunsPolicy.rejectedExecution(r, executor);
}
); );
this.POISON = POISON; this.POISON = POISON;
} }
public CompletableFuture<Void> doneFuture() {
return done;
}
public void startRunning() { public void startRunning() {
ioPool.submit(this); ioPool.submit(this);
} }
@ -78,12 +80,12 @@ public class Stage<X extends StagePayload, Z extends StagePayload> implements Ru
} }
putQuiet(out, POISON); putQuiet(out, POISON);
log.info("stage {} finished.", dispatcher.getClass().getSimpleName()); log.info("stage {} finished.", dispatcher.getClass().getSimpleName());
done.complete(null);
return; return;
} }
counter++; counter++;
int finalCounter = counter; int finalCounter = counter;
Future<Z> f = ioPool.submit(() -> { Future<Z> f = ioPool.submit(() -> {
try {
if (wasInterrupted()) { if (wasInterrupted()) {
log.info("stage {} task {} interrupted...", dispatcher.getClass().getSimpleName(), finalCounter); log.info("stage {} task {} interrupted...", dispatcher.getClass().getSimpleName(), finalCounter);
return null; return null;
@ -91,24 +93,35 @@ public class Stage<X extends StagePayload, Z extends StagePayload> implements Ru
log.info("stage {} task {} started.", dispatcher.getClass().getSimpleName(), finalCounter); log.info("stage {} task {} started.", dispatcher.getClass().getSimpleName(), finalCounter);
Z p = dispatcher.handle(b); Z p = dispatcher.handle(b);
putQuiet(out, p); putQuiet(out, p);
log.info("stage {} task {} finished.", dispatcher.getClass().getSimpleName(), finalCounter); log.info("stage {} task {} ended.", dispatcher.getClass().getSimpleName(), finalCounter);
return p; return p;
} catch (Exception e) {
log.error("{} ERROR: {}", dispatcherName, ExceptionUtils.getStackTrace(e));
return null;
}
} }
); );
inflight.add(f); inflight.add(f);
} }
} catch (Exception e) { } catch (Exception e) {
log.error("error: {}", ExceptionUtils.getStackTrace(e)); log.error("Error for {}: {}; {}", dispatcherName, info(), ExceptionUtils.getStackTrace(e));
putQuiet(out, POISON); putQuiet(out, POISON);
done.complete(null);
} }
} }
private String info() {
ThreadPoolExecutor casted = (ThreadPoolExecutor) ioPool;
return "executor active tasks: %d; queue size: %d, completed tasks: %d".formatted(
casted.getActiveCount(),
casted.getQueue().size(),
casted.getCompletedTaskCount()
);
}
private boolean wasInterrupted() { private boolean wasInterrupted() {
if (Thread.currentThread().isInterrupted()) { if (Thread.currentThread().isInterrupted()) {
log.warn("pipeline was interrupted..."); log.warn("pipeline was interrupted...");
putQuiet(out, POISON); putQuiet(out, POISON);
done.complete(null);
return true; return true;
} }
return false; return false;
@ -117,9 +130,6 @@ public class Stage<X extends StagePayload, Z extends StagePayload> implements Ru
public void close() { public void close() {
log.info("Stage {} is getting shutdown", dispatcher.getClass().getSimpleName()); log.info("Stage {} is getting shutdown", dispatcher.getClass().getSimpleName());
ioPool.shutdownNow(); // прерываем обработчики ioPool.shutdownNow(); // прерываем обработчики
if (!done.isDone()) {
done.completeExceptionally(new java.util.concurrent.CancellationException("Stage cancelled"));
}
} }
void putQuiet(BlockingQueue<Z> q, Z b) { void putQuiet(BlockingQueue<Z> q, Z b) {
@ -129,7 +139,7 @@ public class Stage<X extends StagePayload, Z extends StagePayload> implements Ru
} }
q.put(b); q.put(b);
} catch (InterruptedException e) { } catch (InterruptedException e) {
log.error("Error: {}", ExceptionUtils.getStackTrace(e)); log.error("Error for {}: {}; {}", dispatcherName, info(), ExceptionUtils.getStackTrace(e));
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
} }
} }
@ -138,7 +148,7 @@ public class Stage<X extends StagePayload, Z extends StagePayload> implements Ru
try { try {
return q.poll(timeout.toMillis(), TimeUnit.MILLISECONDS); return q.poll(timeout.toMillis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) { } catch (InterruptedException e) {
log.error("Error: {}", ExceptionUtils.getStackTrace(e)); log.error("Error for {}: {}; {}", dispatcherName, info(), ExceptionUtils.getStackTrace(e));
Thread.currentThread().interrupt(); Thread.currentThread().interrupt();
return null; return null;
} }

10
src/main/java/ru/nbch/credit_tracker/service/task/impl/KickPipeLineImpl.java
View file

@ -15,6 +15,7 @@ import ru.nbch.credit_tracker.model.SubjectProfile;
import ru.nbch.credit_tracker.model.impl.SubjectPayload; import ru.nbch.credit_tracker.model.impl.SubjectPayload;
import ru.nbch.credit_tracker.service.task.setup.IPipeLineStarter; import ru.nbch.credit_tracker.service.task.setup.IPipeLineStarter;
import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor; import ru.nbch.credit_tracker.service.xml.stax.StaxXmlProcessor;
import ru.nbch.credit_tracker.utils.error.ExceptionUtils;
public class KickPipeLineImpl implements IPipeLineStarter<SubjectPayload> { public class KickPipeLineImpl implements IPipeLineStarter<SubjectPayload> {
private final Logger log = LoggerFactory.getLogger(getClass()); private final Logger log = LoggerFactory.getLogger(getClass());
@ -51,6 +52,7 @@ public class KickPipeLineImpl implements IPipeLineStarter<SubjectPayload> {
try { try {
firstQueue.put(new SubjectPayload(packageId, subjectProfileBatch)); firstQueue.put(new SubjectPayload(packageId, subjectProfileBatch));
} catch (InterruptedException e) { } catch (InterruptedException e) {
log.error("{}", ExceptionUtils.getStackTrace(e));
throw new RuntimeException(e); throw new RuntimeException(e);
} }
request.getSubjects().getSubject().clear(); request.getSubjects().getSubject().clear();
@ -60,10 +62,14 @@ public class KickPipeLineImpl implements IPipeLineStarter<SubjectPayload> {
, ,
BATCH_SIZE, "/MonitoringRequest/Subjects/Subject"); BATCH_SIZE, "/MonitoringRequest/Subjects/Subject");
} catch (IOException e) { } catch (IOException e) {
throw new RuntimeException("Error while parsing monitoring request", e); log.error("{}", ExceptionUtils.getStackTrace(e));
} finally { } finally {
log.debug("Send poison pill"); log.debug("Send poison pill");
firstQueue.add(new SubjectPayload(true)); try {
firstQueue.put(new SubjectPayload(true));
} catch (InterruptedException e) {
log.error("couldn't send poison: {}", ExceptionUtils.getStackTrace(e));
}
} }
} }
} }