My Ktor server is randomly giving exception as shown below. I am using rate limiter plugin and status pages plugin along with a custom onCall plugin. I am not able to get exactly when this is getting thrown because sometimes its throwing it and sometimes it is not throwing any exception.
var blockedIP: MutableList<String> = mutableListOf<String>()
var badAttempts: MutableMap<String, Int> = mutableMapOf<String, Int>()
val URLCheckPathPlugin = createApplicationPlugin("URLCheckPathPlugin") {
onCall { call ->
printMyStr("### Checking URL path is correct for new client request ###")
call.request.origin.apply {
//printMyStr("Request URL: $scheme://$localHost:$localPort$uri")
printMyStr("Client details: $remoteAddress ++ $remoteHost ++ $remotePort")
printMyStr("URL: ${method.value} ++ $version ++ $scheme://$serverHost:${serverPort.toString()}$uri")
if (blockedIP.contains(remoteAddress))
{
printMyStr("!!! WARN: Client address is blocked !!!")
call.respond(HttpStatusCode.BadRequest, "")
}
else
{
if(uri == "/eod")
{
printMyStr("%%% Received EOD message %%%")
printMyStr("BlockedIP: ${blockedIP.toString()}")
blockedIP.clear()
printMyStr("BadAttempts: ${badAttempts.toString()}")
badAttempts.clear()
}
else
{
if (badAttempts.contains(remoteAddress))
badAttempts[remoteAddress] = badAttempts.getValue(remoteAddress) + 1
else
badAttempts[remoteAddress] = 1
if(badAttempts.getValue(remoteAddress) == 10)
blockedIP.add(remoteAddress)
printMyStr("*** ERROR: Since this page does not exist responding with page not found #${badAttempts.getValue(remoteAddress)} ***")
//myHTTPrespond(call, 404, "Page not found!", "ERROR", "") //dont do this it will cause exceptions
call.response.status(HttpStatusCode(404, "Page not found!"))
call.response.header("ERROR", "")
}
}
}
}
val URLFailPlugin = createApplicationPlugin("URLFailPlugin") {
on(CallFailed) { call, error ->
//call.request.origin.apply {
printMyStr("*** Client request failed to connect to server ***")
printMyStr("Client details: $call.remoteAddress ++ $call.remoteHost ++ $call.remotePort ***")
printMyStr("URL: $call.method.value ++ $call.version ++ $call.scheme://$call.serverHost:$call.serverPort.toString()$call.uri")
printMyStr("ERROR: ${error.message}")
//}
}
}
fun Application.configureURLRequestValidation()
{
install(RateLimit) {
global {
rateLimiter(limit = 10, refillPeriod = 60.seconds)
requestKey { call -> call.request.origin.remoteAddress }
}
}
install(StatusPages) {
status(HttpStatusCode.TooManyRequests) { call, status ->
blockedIP.add(call.request.origin.remoteAddress)
printMyStr("*** ERROR: Blocking this client for too many requests ***")
call.respondText(text = "429: Too many requests. You are now blocked.", status = status)
}
}
install(URLFailPlugin)
install(URLCheckPathPlugin)
}
Exception in thread "eventLoopGroupProxy-3-1" java.util.concurrent.RejectedExecutionException: event executor terminated
at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:934)
at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:351)
at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:344)
at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:836)
at io.netty.util.concurrent.SingleThreadEventExecutor.execute0(SingleThreadEventExecutor.java:827)
at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:817)
at io.ktor.server.netty.NettyDispatcher.dispatch(CIO.kt:68)
at kotlinx.coroutines.internal.DispatchedContinuation.resumeWith(DispatchedContinuation.kt:201)
at io.ktor.utils.io.internal.CancellableReusableContinuation.resumeWith(CancellableReusableContinuation.kt:93)
at io.ktor.utils.io.ByteBufferChannel.resumeReadOp(ByteBufferChannel.kt:2097)
at io.ktor.utils.io.ByteBufferChannel.tryTerminate$ktor_io(ByteBufferChannel.kt:388)
at io.ktor.utils.io.ByteBufferChannel.close(ByteBufferChannel.kt:133)
at io.ktor.server.netty.http2.HttpFrameAdapterKt.http2frameLoop(HttpFrameAdapter.kt:41)
at io.ktor.server.netty.http2.HttpFrameAdapterKt$http2frameLoop$1.invokeSuspend(HttpFrameAdapter.kt)
at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
at kotlinx.coroutines.DispatchedTaskKt.resume(DispatchedTask.kt:234)
at kotlinx.coroutines.DispatchedTaskKt.resumeUnconfined(DispatchedTask.kt:190)
at kotlinx.coroutines.DispatchedTaskKt.dispatch(DispatchedTask.kt:161)
at kotlinx.coroutines.CancellableContinuationImpl.dispatchResume(CancellableContinuationImpl.kt:397)
at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl(CancellableContinuationImpl.kt:431)
at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl$default(CancellableContinuationImpl.kt:420)
at kotlinx.coroutines.CancellableContinuationImpl.resumeWith(CancellableContinuationImpl.kt:328)
at kotlinx.coroutines.channels.AbstractChannel$ReceiveElement.resumeReceiveClosed(AbstractChannel.kt:913)
at kotlinx.coroutines.channels.AbstractSendChannel.helpClose(AbstractChannel.kt:342)
at kotlinx.coroutines.channels.AbstractSendChannel.close(AbstractChannel.kt:271)
at kotlinx.coroutines.channels.AbstractChannel.cancelInternal$kotlinx_coroutines_core(AbstractChannel.kt:661)
at kotlinx.coroutines.channels.AbstractChannel.cancel(AbstractChannel.kt:656)
at kotlinx.coroutines.channels.ActorCoroutine.onCancelling(Actor.kt:137)
at kotlinx.coroutines.JobSupport.notifyCancelling(JobSupport.kt:329)
at kotlinx.coroutines.JobSupport.tryMakeCancelling(JobSupport.kt:795)
at kotlinx.coroutines.JobSupport.makeCancelling(JobSupport.kt:755)
at kotlinx.coroutines.JobSupport.cancelImpl$kotlinx_coroutines_core(JobSupport.kt:671)
at kotlinx.coroutines.JobSupport.parentCancelled(JobSupport.kt:637)
at kotlinx.coroutines.ChildHandleNode.invoke(JobSupport.kt:1466)
at kotlinx.coroutines.JobSupport.notifyCancelling(JobSupport.kt:1500)
at kotlinx.coroutines.JobSupport.tryMakeCompletingSlowPath(JobSupport.kt:900)
at kotlinx.coroutines.JobSupport.tryMakeCompleting(JobSupport.kt:863)
at kotlinx.coroutines.JobSupport.cancelMakeCompleting(JobSupport.kt:696)
at kotlinx.coroutines.JobSupport.cancelImpl$kotlinx_coroutines_core(JobSupport.kt:667)
at kotlinx.coroutines.JobSupport.cancelInternal(JobSupport.kt:632)
at kotlinx.coroutines.JobSupport.cancel(JobSupport.kt:617)
at kotlinx.coroutines.CoroutineScopeKt.cancel(CoroutineScope.kt:287)
at kotlinx.coroutines.CoroutineScopeKt.cancel$default(CoroutineScope.kt:285)
at io.ktor.server.netty.NettyChannelInitializer.configurePipeline$lambda$5(NettyChannelInitializer.kt:119)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625)
at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105)
at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)
at io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1164)
at io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:755)
at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)
at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)
at io.netty.channel.nio.NioEventLoop.closeAll(NioEventLoop.java:838)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:595)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.ktor.server.netty.EventLoopGroupProxy$Companion.create$lambda$1$lambda$0(NettyApplicationEngine.kt:296)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:750)
Suppressed: kotlinx.coroutines.DiagnosticCoroutineContextException: [ActorCoroutine{Cancelling}@2cf803c3, Dispatchers.Unconfined]
Exception in thread "eventLoopGroupProxy-3-1" java.util.concurrent.RejectedExecutionException: event executor terminated
at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:934)
at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:351)
at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:344)
at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:836)
at io.netty.util.concurrent.SingleThreadEventExecutor.execute0(SingleThreadEventExecutor.java:827)
at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:817)
at io.ktor.server.netty.NettyDispatcher.dispatch(CIO.kt:68)
at kotlinx.coroutines.internal.DispatchedContinuation.resumeWith(DispatchedContinuation.kt:201)
at io.ktor.utils.io.internal.CancellableReusableContinuation.resumeWith(CancellableReusableContinuation.kt:93)
at io.ktor.utils.io.ByteBufferChannel.resumeReadOp(ByteBufferChannel.kt:2097)
at io.ktor.utils.io.ByteBufferChannel.tryTerminate$ktor_io(ByteBufferChannel.kt:388)
at io.ktor.utils.io.ByteBufferChannel.close(ByteBufferChannel.kt:133)
at io.ktor.server.netty.http2.HttpFrameAdapterKt.http2frameLoop(HttpFrameAdapter.kt:41)
at io.ktor.server.netty.http2.HttpFrameAdapterKt$http2frameLoop$1.invokeSuspend(HttpFrameAdapter.kt)
at kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:33)
at kotlinx.coroutines.DispatchedTaskKt.resume(DispatchedTask.kt:234)
at kotlinx.coroutines.DispatchedTaskKt.resumeUnconfined(DispatchedTask.kt:190)
at kotlinx.coroutines.DispatchedTaskKt.dispatch(DispatchedTask.kt:161)
at kotlinx.coroutines.CancellableContinuationImpl.dispatchResume(CancellableContinuationImpl.kt:397)
at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl(CancellableContinuationImpl.kt:431)
at kotlinx.coroutines.CancellableContinuationImpl.resumeImpl$default(CancellableContinuationImpl.kt:420)
at kotlinx.coroutines.CancellableContinuationImpl.resumeWith(CancellableContinuationImpl.kt:328)
at kotlinx.coroutines.channels.AbstractChannel$ReceiveElement.resumeReceiveClosed(AbstractChannel.kt:913)
at kotlinx.coroutines.channels.AbstractSendChannel.helpClose(AbstractChannel.kt:342)
at kotlinx.coroutines.channels.AbstractSendChannel.close(AbstractChannel.kt:271)
at kotlinx.coroutines.channels.AbstractChannel.cancelInternal$kotlinx_coroutines_core(AbstractChannel.kt:661)
at kotlinx.coroutines.channels.AbstractChannel.cancel(AbstractChannel.kt:656)
at kotlinx.coroutines.channels.ActorCoroutine.onCancelling(Actor.kt:137)
at kotlinx.coroutines.JobSupport.notifyCancelling(JobSupport.kt:329)
at kotlinx.coroutines.JobSupport.tryMakeCancelling(JobSupport.kt:795)
at kotlinx.coroutines.JobSupport.makeCancelling(JobSupport.kt:755)
at kotlinx.coroutines.JobSupport.cancelImpl$kotlinx_coroutines_core(JobSupport.kt:671)
at kotlinx.coroutines.JobSupport.parentCancelled(JobSupport.kt:637)
at kotlinx.coroutines.ChildHandleNode.invoke(JobSupport.kt:1466)
at kotlinx.coroutines.JobSupport.notifyCancelling(JobSupport.kt:1500)
at kotlinx.coroutines.JobSupport.tryMakeCompletingSlowPath(JobSupport.kt:900)
at kotlinx.coroutines.JobSupport.tryMakeCompleting(JobSupport.kt:863)
at kotlinx.coroutines.JobSupport.cancelMakeCompleting(JobSupport.kt:696)
at kotlinx.coroutines.JobSupport.cancelImpl$kotlinx_coroutines_core(JobSupport.kt:667)
at kotlinx.coroutines.JobSupport.cancelInternal(JobSupport.kt:632)
at kotlinx.coroutines.JobSupport.cancel(JobSupport.kt:617)
at kotlinx.coroutines.CoroutineScopeKt.cancel(CoroutineScope.kt:287)
at kotlinx.coroutines.CoroutineScopeKt.cancel$default(CoroutineScope.kt:285)
at io.ktor.server.netty.NettyChannelInitializer.configurePipeline$lambda$5(NettyChannelInitializer.kt:119)
at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590)
at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557)
at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492)
at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636)
at io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625)
at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105)
at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84)
at io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1164)
at io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:755)
at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:731)
at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:620)
at io.netty.channel.nio.NioEventLoop.closeAll(NioEventLoop.java:838)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:595)
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
at io.ktor.server.netty.EventLoopGroupProxy$Companion.create$lambda$1$lambda$0(NettyApplicationEngine.kt:296)
at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
at java.lang.Thread.run(Thread.java:750)
Suppressed: kotlinx.coroutines.DiagnosticCoroutineContextException: [ActorCoroutine{Cancelling}@770a75ee, Dispatchers.Unconfined]
Any idea what may be causing this exception and how to fix it?