-
Bug
-
Resolution: Fixed
-
Critical
-
workflow-api-plugin v2.33
Jenkins v2.150.3
-
-
workflow-api 1239.vd7c497375cb_f
Some jobs get stuck in a state where they're unable to continue to write data to the console log. The system log contains multiple entries of the following form until the job ends:
failed to flush /net/users/jenkins/configuration/jobs/XXX/builds/5/log
java.io.IOException: Stream Closed
at java.io.FileOutputStream.writeBytes(Native Method)
at java.io.FileOutputStream.write(FileOutputStream.java:326)
at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$FlushControlledOutputStream.write(DelayBufferedOutputStream.java:134)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at org.jenkinsci.plugins.workflow.log.FileLogStorage.maybeFlush(FileLogStorage.java:190)
at org.jenkinsci.plugins.workflow.log.FileLogStorage.overallLog(FileLogStorage.java:198)
at org.jenkinsci.plugins.workflow.job.WorkflowRun.getLogText(WorkflowRun.java:1018)
at org.jenkinsci.plugins.workflow.job.WorkflowRun.getLogInputStream(WorkflowRun.java:1026)
at org.jenkinsci.plugins.gwt.GenericWebhookEnvironmentContributor.notLogged(GenericWebhookEnvironmentContributor.java:54)
at org.jenkinsci.plugins.gwt.GenericWebhookEnvironmentContributor.buildEnvironmentFor(GenericWebhookEnvironmentContributor.java:30)
at hudson.model.Run.getEnvironment(Run.java:2373)
at org.jenkinsci.plugins.workflow.job.WorkflowRun.getEnvironment(WorkflowRun.java:468)
at org.jenkinsci.plugins.workflow.cps.EnvActionImpl.getEnvironment(EnvActionImpl.java:86)
at org.jenkinsci.plugins.workflow.cps.EnvActionImpl.getEnvironment(EnvActionImpl.java:67)
at org.jenkinsci.plugins.workflow.support.DefaultStepContext.get(DefaultStepContext.java:72)
at org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep$Execution.start(DurableTaskStep.java:305)
at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:268)
at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:176)
at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:122)
at sun.reflect.GeneratedMethodAccessor387.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93)
at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325)
at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1213)
at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1022)
at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:42)
at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:48)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:113)
at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:157)
at org.kohsuke.groovy.sandbox.GroovyInterceptor.onMethodCall(GroovyInterceptor.java:23)
at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onMethodCall(SandboxInterceptor.java:155)
at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:155)
at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:159)
at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.methodCall(SandboxInvoker.java:17)
at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:57)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:109)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:82)
at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:103)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:82)
at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:60)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:109)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:82)
at sun.reflect.GeneratedMethodAccessor384.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
at com.cloudbees.groovy.cps.impl.ConstantBlock.eval(ConstantBlock.java:21)
at com.cloudbees.groovy.cps.Next.step(Next.java:83)
at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:174)
at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:163)
at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:129)
at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:268)
at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:163)
at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$101(SandboxContinuable.java:34)
at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.lambda$run0$0(SandboxContinuable.java:59)
at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.GroovySandbox.runInSandbox(GroovySandbox.java:121)
at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:58)
at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:182)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:332)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.access$200(CpsThreadGroup.java:83)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:244)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:232)
at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:64)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:131)
at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:59)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
When enabling DelayBufferedOutputStream logging, we also see this in the log:
null
java.io.IOException: Stream Closed
at java.io.FileOutputStream.writeBytes(Native Method)
at java.io.FileOutputStream.write(FileOutputStream.java:326)
at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$FlushControlledOutputStream.write(DelayBufferedOutputStream.java:134)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream.flushBuffer(DelayBufferedOutputStream.java:82)
at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream.flushAndReschedule(DelayBufferedOutputStream.java:91)
at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$Flush.run(DelayBufferedOutputStream.java:114)
at jenkins.security.ImpersonatingScheduledExecutorService$1.run(ImpersonatingScheduledExecutorService.java:58)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
- is duplicated by
-
-
- Closed
-
- relates to
-
-
- Open
-
[JENKINS-56446] IOException in FileLogStorage maybeFlush
Thierry GATINEAU
added a comment - I can reproduce this issue when I uses logstash plugin inside a pipeline.
See https://issues.jenkins-ci.org/browse/JENKINS-62354
Thierry GATINEAU
added a comment - I can reproduce this issue when I uses logstash plugin inside a pipeline.
See https://issues.jenkins-ci.org/browse/JENKINS-62354
John Lengeling
added a comment - Any update or progress on this issue? I am running into this under Jenkins 2.163.3 and workflow-api 2.41.
John Lengeling
added a comment - Any update or progress on this issue? I am running into this under Jenkins 2.163.3 and workflow-api 2.41. Observed with jenkins:2.235.5 and workflow-api:2.41, durable-task:1.34
John Lengeling
added a comment - We just recently started seeing a large number of maybeFlush errors across multiple Jenkins controllers. This issue showed up after our Jenkins controllers were migrated from on on-prem VMs to Amazon AWS ECS. Jenkins was running natively on the VMs and were moved running in docker on AWS ECS. The ECS EC2 nodes are configured without swap space.
We seem to have reduced the occurrence of the issue by increasing the amount of ram and heap allocated to the Jenkins docker container. We increased the heap from 6G->9G and memory from 12G to 18G. This seems to have helped but we still have a few systems run into the maybeFlush message and missing console log output. So more memory/heap seems to have helped a bit.
We are running jenkins:2.263.3, workflow-api:2.42
We have also tried running with the logstash plugin uninstalled but that didn't help.
We have tried enabling swap on the ECS EC2 nodes and in the docker container but that didn't help.
We have a job that can reliably duplicate the problem and can help in debugging this issue or testing a change.
John Lengeling
added a comment - We just recently started seeing a large number of maybeFlush errors across multiple Jenkins controllers. This issue showed up after our Jenkins controllers were migrated from on on-prem VMs to Amazon AWS ECS. Jenkins was running natively on the VMs and were moved running in docker on AWS ECS. The ECS EC2 nodes are configured without swap space.
We seem to have reduced the occurrence of the issue by increasing the amount of ram and heap allocated to the Jenkins docker container. We increased the heap from 6G->9G and memory from 12G to 18G. This seems to have helped but we still have a few systems run into the maybeFlush message and missing console log output. So more memory/heap seems to have helped a bit.
We are running jenkins:2.263.3, workflow-api:2.42
We have also tried running with the logstash plugin uninstalled but that didn't help.
We have tried enabling swap on the ECS EC2 nodes and in the docker container but that didn't help.
We have a job that can reliably duplicate the problem and can help in debugging this issue or testing a change. What storage are you all using for JENKINS_HOME? If NFS, please share your mount options.
Christopher Head
added a comment - I see this problem. I’m not using logstash. I’m not using NFS; storage is a local btrfs filesystem. After the problem shows up, restarting Jenkins seems to clear it up, in the sense that jobs keep running (as they usually do across a restart) but the missing output arrives and the console view starts showing live output again.
Christopher Head
added a comment - I see this problem. I’m not using logstash. I’m not using NFS; storage is a local btrfs filesystem. After the problem shows up, restarting Jenkins seems to clear it up, in the sense that jobs keep running (as they usually do across a restart) but the missing output arrives and the console view starts showing live output again. haw777 thanks for your input. I wonder if the btrfs filesystem could be a contributor here. We've wondered if the root cause of this is a transient IO performance issue where writes get slowed down or blocked enough to then screw up or overload a buffer in Jenkins. The original stack trace in this bug has a reference to the Generic Webhook plugin in addition to the expected references to the Pipeline plugins. In another case I have seen references to the Splunk plugin, and in that case the issue was greatly reduced (though not eliminated) by downgrading the version of that. Can you (and anyone else with this issue) share the exact stack trace you are seeing?
Christopher Head
added a comment - I can definitely confirm that, when the problem happened, the machine in question was in a state where it suffered slower than normal I/O performance due to a hardware issue. Here’s my stack trace.
2021-11-30 04:11:42.956+0000 [id=58892] WARNING o.j.p.w.log.FileLogStorage#maybeFlush: failed to flush /var/lib/jenkins/jobs/<REDACTED>/branches/<REDACTED>/builds/2/log java.io.IOException: Stream Closed at java.base/java.io.FileOutputStream.writeBytes(Native Method) at java.base/java.io.FileOutputStream.write(FileOutputStream.java:354) at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$FlushControlledOutputStream.write(DelayBufferedOutputStream.java:125) at java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) at java.base/java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) at java.base/java.io.FilterOutputStream.flush(FilterOutputStream.java:153) at org.jenkinsci.plugins.workflow.log.FileLogStorage.maybeFlush(FileLogStorage.java:190) at org.jenkinsci.plugins.workflow.log.FileLogStorage.overallLog(FileLogStorage.java:198) at org.jenkinsci.plugins.workflow.job.WorkflowRun.getLogText(WorkflowRun.java:1065) at jdk.internal.reflect.GeneratedMethodAccessor1058.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:566) at org.apache.commons.jexl.util.PropertyExecutor.execute(PropertyExecutor.java:125) at org.apache.commons.jexl.util.introspection.UberspectImpl$VelGetterImpl.invoke(UberspectImpl.java:314) at org.apache.commons.jexl.parser.ASTArrayAccess.evaluateExpr(ASTArrayAccess.java:185) at org.apache.commons.jexl.parser.ASTIdentifier.execute(ASTIdentifier.java:75) at org.apache.commons.jexl.parser.ASTReference.execute(ASTReference.java:83) at org.apache.commons.jexl.parser.ASTReference.value(ASTReference.java:57) at org.apache.commons.jexl.parser.ASTGTNode.value(ASTGTNode.java:62) at org.apache.commons.jexl.parser.ASTExpression.value(ASTExpression.java:54) at org.apache.commons.jexl.parser.ASTExpressionExpression.value(ASTExpressionExpression.java:56) at org.apache.commons.jexl.ExpressionImpl.evaluate(ExpressionImpl.java:80) at hudson.ExpressionFactory2$JexlExpression.evaluate(ExpressionFactory2.java:76) at org.apache.commons.jelly.expression.ExpressionSupport.evaluateRecurse(ExpressionSupport.java:61) at org.apache.commons.jelly.expression.ExpressionSupport.evaluateAsBoolean(ExpressionSupport.java:71) at org.apache.commons.jelly.tags.core.WhenTag.doTag(WhenTag.java:44) at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.apache.commons.jelly.TagSupport.invokeBody(TagSupport.java:161) at org.apache.commons.jelly.tags.core.ChooseTag.doTag(ChooseTag.java:38) at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:99) at org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91) at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269) at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100) at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105) at org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:120) at org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:99) at org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91) at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269) at org.apache.commons.jelly.tags.core.CoreTagLibrary$1.run(CoreTagLibrary.java:98) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105) at org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:120) at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105) at org.kohsuke.stapler.jelly.JellyViewScript.run(JellyViewScript.java:95) at org.kohsuke.stapler.jelly.IncludeTag.doTag(IncludeTag.java:171) at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:99) at org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91) at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100) at org.apache.commons.jelly.tags.core.CoreTagLibrary$1.run(CoreTagLibrary.java:98) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:99) at org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91) at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105) at org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:120) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105) at org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:120) at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95) at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105) at org.kohsuke.stapler.jelly.JellyViewScript.run(JellyViewScript.java:95) at org.kohsuke.stapler.jelly.DefaultScriptInvoker.invokeScript(DefaultScriptInvoker.java:64) at org.kohsuke.stapler.jelly.DefaultScriptInvoker.invokeScript(DefaultScriptInvoker.java:54) at org.kohsuke.stapler.jelly.ScriptInvoker.execute(ScriptInvoker.java:56) at org.kohsuke.stapler.jelly.ScriptInvoker.execute(ScriptInvoker.java:43) at org.kohsuke.stapler.Facet.handleIndexRequest(Facet.java:284) at org.kohsuke.stapler.jelly.JellyFacet.handleIndexRequest(JellyFacet.java:100) at org.kohsuke.stapler.IndexViewDispatcher.dispatch(IndexViewDispatcher.java:32) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.MetaClass$9.dispatch(MetaClass.java:457) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:281) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:281) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:694) at org.kohsuke.stapler.Stapler.service(Stapler.java:240) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799) at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:156) at org.jenkinsci.plugins.ssegateway.Endpoint$SSEListenChannelFilter.doFilter(Endpoint.java:248) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153) at jenkins.telemetry.impl.UserLanguages$AcceptLanguageFilter.doFilter(UserLanguages.java:128) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153) at jenkins.security.ResourceDomainFilter.doFilter(ResourceDomainFilter.java:80) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153) at io.jenkins.blueocean.ResourceCacheControl.doFilter(ResourceCacheControl.java:134) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153) at io.jenkins.blueocean.auth.jwt.impl.JwtAuthenticationFilter.doFilter(JwtAuthenticationFilter.java:60) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153) at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:159) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:159) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:92) at jenkins.security.AcegiSecurityExceptionFilter.doFilter(AcegiSecurityExceptionFilter.java:52) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:53) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:121) at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:115) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:105) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:133) at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:92) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:218) at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:212) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.java:97) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:110) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:80) at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:62) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:109) at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:171) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:51) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:85) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at jenkins.security.SuspiciousRequestFilter.doFilter(SuspiciousRequestFilter.java:35) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:386) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base/java.lang.Thread.run(Thread.java:829)
Christopher Head
added a comment - I can definitely confirm that, when the problem happened, the machine in question was in a state where it suffered slower than normal I/O performance due to a hardware issue. Here’s my stack trace.
2021-11-30 04:11:42.956+0000 [id=58892] WARNING o.j.p.w.log.FileLogStorage#maybeFlush: failed to flush /var/lib/jenkins/jobs/<REDACTED>/branches/<REDACTED>/builds/2/log
java.io.IOException: Stream Closed
at java.base/java.io.FileOutputStream.writeBytes(Native Method)
at java.base/java.io.FileOutputStream.write(FileOutputStream.java:354)
at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$FlushControlledOutputStream.write(DelayBufferedOutputStream.java:125)
at java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81)
at java.base/java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142)
at java.base/java.io.FilterOutputStream.flush(FilterOutputStream.java:153)
at org.jenkinsci.plugins.workflow.log.FileLogStorage.maybeFlush(FileLogStorage.java:190)
at org.jenkinsci.plugins.workflow.log.FileLogStorage.overallLog(FileLogStorage.java:198)
at org.jenkinsci.plugins.workflow.job.WorkflowRun.getLogText(WorkflowRun.java:1065)
at jdk.internal.reflect.GeneratedMethodAccessor1058.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.apache.commons.jexl.util.PropertyExecutor.execute(PropertyExecutor.java:125)
at org.apache.commons.jexl.util.introspection.UberspectImpl$VelGetterImpl.invoke(UberspectImpl.java:314)
at org.apache.commons.jexl.parser.ASTArrayAccess.evaluateExpr(ASTArrayAccess.java:185)
at org.apache.commons.jexl.parser.ASTIdentifier.execute(ASTIdentifier.java:75)
at org.apache.commons.jexl.parser.ASTReference.execute(ASTReference.java:83)
at org.apache.commons.jexl.parser.ASTReference.value(ASTReference.java:57)
at org.apache.commons.jexl.parser.ASTGTNode.value(ASTGTNode.java:62)
at org.apache.commons.jexl.parser.ASTExpression.value(ASTExpression.java:54)
at org.apache.commons.jexl.parser.ASTExpressionExpression.value(ASTExpressionExpression.java:56)
at org.apache.commons.jexl.ExpressionImpl.evaluate(ExpressionImpl.java:80)
at hudson.ExpressionFactory2$JexlExpression.evaluate(ExpressionFactory2.java:76)
at org.apache.commons.jelly.expression.ExpressionSupport.evaluateRecurse(ExpressionSupport.java:61)
at org.apache.commons.jelly.expression.ExpressionSupport.evaluateAsBoolean(ExpressionSupport.java:71)
at org.apache.commons.jelly.tags.core.WhenTag.doTag(WhenTag.java:44)
at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.apache.commons.jelly.TagSupport.invokeBody(TagSupport.java:161)
at org.apache.commons.jelly.tags.core.ChooseTag.doTag(ChooseTag.java:38)
at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:99)
at org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269)
at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100)
at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
at org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:120)
at org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:99)
at org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269)
at org.apache.commons.jelly.tags.core.CoreTagLibrary$1.run(CoreTagLibrary.java:98)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
at org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:120)
at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
at org.kohsuke.stapler.jelly.JellyViewScript.run(JellyViewScript.java:95)
at org.kohsuke.stapler.jelly.IncludeTag.doTag(IncludeTag.java:171)
at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:99)
at org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100)
at org.apache.commons.jelly.tags.core.CoreTagLibrary$1.run(CoreTagLibrary.java:98)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:100)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:99)
at org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
at org.apache.commons.jelly.impl.TagScript.run(TagScript.java:269)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
at org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:120)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
at org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:120)
at org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
at org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
at org.kohsuke.stapler.jelly.JellyViewScript.run(JellyViewScript.java:95)
at org.kohsuke.stapler.jelly.DefaultScriptInvoker.invokeScript(DefaultScriptInvoker.java:64)
at org.kohsuke.stapler.jelly.DefaultScriptInvoker.invokeScript(DefaultScriptInvoker.java:54)
at org.kohsuke.stapler.jelly.ScriptInvoker.execute(ScriptInvoker.java:56)
at org.kohsuke.stapler.jelly.ScriptInvoker.execute(ScriptInvoker.java:43)
at org.kohsuke.stapler.Facet.handleIndexRequest(Facet.java:284)
at org.kohsuke.stapler.jelly.JellyFacet.handleIndexRequest(JellyFacet.java:100)
at org.kohsuke.stapler.IndexViewDispatcher.dispatch(IndexViewDispatcher.java:32)
at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898)
at org.kohsuke.stapler.MetaClass$9.dispatch(MetaClass.java:457)
at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898)
at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:281)
at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58)
at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898)
at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:281)
at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58)
at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:694)
at org.kohsuke.stapler.Stapler.service(Stapler.java:240)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799)
at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:156)
at org.jenkinsci.plugins.ssegateway.Endpoint$SSEListenChannelFilter.doFilter(Endpoint.java:248)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153)
at jenkins.telemetry.impl.UserLanguages$AcceptLanguageFilter.doFilter(UserLanguages.java:128)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153)
at jenkins.security.ResourceDomainFilter.doFilter(ResourceDomainFilter.java:80)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153)
at io.jenkins.blueocean.ResourceCacheControl.doFilter(ResourceCacheControl.java:134)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153)
at io.jenkins.blueocean.auth.jwt.impl.JwtAuthenticationFilter.doFilter(JwtAuthenticationFilter.java:60)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153)
at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:159)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:159)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:92)
at jenkins.security.AcegiSecurityExceptionFilter.doFilter(AcegiSecurityExceptionFilter.java:52)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:53)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:121)
at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:115)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:105)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:133)
at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:92)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:218)
at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:212)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.java:97)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:110)
at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:80)
at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:62)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:109)
at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:171)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:51)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:85)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at jenkins.security.SuspiciousRequestFilter.doFilter(SuspiciousRequestFilter.java:35)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:516)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:386)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
at java.base/java.lang.Thread.run(Thread.java:829)
haw777 thanks for that. In your case I only see references to Pipeline (workflow) and the SSE Gateway plugin, which I think is just coming through via Blue Ocean. I continue to suspect that this issue is caused by a storage performance problem. Whether that issue is triggered by excessive IO from Jenkins (maybe the case with the Splunk and Logstash plugins doing some heavy reads or something) or just unrelated infrastructure problems may be irrelevant. I don't know whether the Jenkins code here can be made more robust. Needs input from a Pipeline maintainer.
Christopher Head
added a comment - I’m not sure how steps actually run under the hood, but it seems to me, if the step writes to a pipe which Jenkins is reading, then if the disk is unable to keep up with writing log data to the job directory then it should stop reading the pipe, which will apply backpressure, eventually slowing down the process producing the output in the first place. On the other hand, if the step writes to a file which Jenkins is tailing, then if the disk is unable to keep up with writing log data to the job directory then it should just read the output file less frequently (i.e. if Jenkins’s buffer is full, then stop reading from the step output file until it drains); this might result in the log data in the Job directory getting significantly behind the process’s live output, but that seems like a much less bad failure mode than completely blowing up and not working at all, which is what we get right now—and eventually, presumably, the running process will end, and thus stop producing output, allowing Jenkins to catch up before starting the next step.
But this is all hypothesizing by someone who really doesn’t know how Jenkins works.
Christopher Head
added a comment - I’m not sure how steps actually run under the hood, but it seems to me, if the step writes to a pipe which Jenkins is reading, then if the disk is unable to keep up with writing log data to the job directory then it should stop reading the pipe, which will apply backpressure, eventually slowing down the process producing the output in the first place. On the other hand, if the step writes to a file which Jenkins is tailing, then if the disk is unable to keep up with writing log data to the job directory then it should just read the output file less frequently (i.e. if Jenkins’s buffer is full, then stop reading from the step output file until it drains); this might result in the log data in the Job directory getting significantly behind the process’s live output, but that seems like a much less bad failure mode than completely blowing up and not working at all, which is what we get right now—and eventually, presumably, the running process will end, and thus stop producing output, allowing Jenkins to catch up before starting the next step.
But this is all hypothesizing by someone who really doesn’t know how Jenkins works. 
mel nal
added a comment - Hi All,
Any update on this? rshade Have you tried any work-around? I have had the same issue described here.
Now after digging around I have found, that the root cause was indeed this issue here: JENKINS-62999
I am using the pipeline tee command in many of our parallel build steps, and it does have a heavy output load as described in JENKINS-62999
My log does look like this:
2022-01-31 13:19:38.241+0000 [id=210] WARNING o.j.r.u.AnonymousClassWarnings#warn: Attempt to (de-)serialize anonymous class org.jenkinsci.plugins.pipeline.utility.steps.fs.TeeStep$1 in file:/home/mrigo/.jenkins/plugins/pipeline-utility-steps/WEB-INF/lib/pipeline-utility-steps.jar; see: https://jenkins.io/redirect/serialization-of-anonymous-classes/ 2022-01-31 13:20:00.194+0000 [id=84] WARNING h.i.i.InstallUncaughtExceptionHandler#handleException: Caught unhandled exception with ID d7135add-6c31-4f76-8331-e8d609127f0b 2022-01-31 13:20:14.451+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 5 sec 2022-01-31 13:20:19.451+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 10 sec 2022-01-31 13:20:24.452+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 15 sec 2022-01-31 13:20:29.452+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 20 sec 2022-01-31 13:20:34.453+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 25 sec 2022-01-31 13:20:39.453+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 30 sec 2022-01-31 13:20:44.453+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 35 sec 2022-01-31 13:20:49.454+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 40 sec 2022-01-31 13:20:54.454+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 45 sec 2022-01-31 13:20:59.454+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 50 sec 2022-01-31 13:21:04.455+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 55 sec 2022-01-31 13:21:08.882+0000 [id=22] WARNING o.j.p.w.log.FileLogStorage#maybeFlush: failed to flush /home/mrigo/.jenkins/jobs/project-Build-pipe/builds/1526/log java.io.IOException: Stream Closed at java.io.FileOutputStream.writeBytes(Native Method) at java.io.FileOutputStream.write(FileOutputStream.java:326) at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$FlushControlledOutputStream.write(DelayBufferedOutputStream.java:127) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.FilterOutputStream.flush(FilterOutputStream.java:140) at org.jenkinsci.plugins.workflow.log.FileLogStorage.maybeFlush(FileLogStorage.java:193) at org.jenkinsci.plugins.workflow.log.FileLogStorage.overallLog(FileLogStorage.java:202) at org.jenkinsci.plugins.workflow.job.WorkflowRun.getLogText(WorkflowRun.java:1065) at java.lang.invoke.MethodHandle.invokeWithArguments(MethodHandle.java:627) at org.kohsuke.stapler.Function$MethodFunction.invoke(Function.java:393) at org.kohsuke.stapler.Function$InstanceFunction.invoke(Function.java:405) at org.kohsuke.stapler.MetaClass$2.doDispatch(MetaClass.java:220) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.MetaClass$9.dispatch(MetaClass.java:457) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:281) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:694) at org.kohsuke.stapler.Stapler.service(Stapler.java:240) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799) at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:156) at jenkins.telemetry.impl.UserLanguages$AcceptLanguageFilter.doFilter(UserLanguages.java:128) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153) at jenkins.security.ResourceDomainFilter.doFilter(ResourceDomainFilter.java:80) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153) at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:239) at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:215) at net.bull.javamelody.PluginMonitoringFilter.doFilter(PluginMonitoringFilter.java:88) at org.jvnet.hudson.plugins.monitoring.HudsonMonitoringFilter.doFilter(HudsonMonitoringFilter.java:121) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153) at jenkins.metrics.impl.MetricsFilter.doFilter(MetricsFilter.java:125) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153) at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:159) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:153) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:92) at jenkins.security.AcegiSecurityExceptionFilter.doFilter(AcegiSecurityExceptionFilter.java:52) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:53) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:121) at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:115) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:105) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:101) at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:92) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:218) at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:212) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.java:97) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:110) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:80) at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:62) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97) at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:109) at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:171) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:51) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:85) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at jenkins.security.SuspiciousRequestFilter.doFilter(SuspiciousRequestFilter.java:39) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:386) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.lang.Thread.run(Thread.java:748)
The reason I was using the pipeline tee was not to mask my make's error output.
tee ('build.log'){ sh 'make -j $(nproc --all)' }
After modifying to this the make's return value still gets passed to the pipe, hence the pipe fails if make fails:
sh """#!/bin/bash set -o pipefail make -j \$(nproc --all) 2>&1 | tee build.log """
Maybe some of you out there also do use tee in pipelines.
Only thing I am wondering is how to do now tee in the windows builds that we do have, as the pipeline tee was actually quiet handy.
Mate Rigo
added a comment - - edited I have had the same issue described here.
Now after digging around I have found, that the root cause was indeed this issue here: JENKINS-62999
I am using the pipeline tee command in many of our parallel build steps, and it does have a heavy output load as described in JENKINS-62999
My log does look like this:
2022-01-31 13:19:38.241+0000 [id=210] WARNING o.j.r.u.AnonymousClassWarnings#warn: Attempt to (de-)serialize anonymous class org.jenkinsci.plugins.pipeline.utility.steps.fs.TeeStep$1 in file:/home/mrigo/.jenkins/plugins/pipeline-utility-steps/WEB-INF/lib/pipeline-utility-steps.jar; see: https: //jenkins.io/redirect/serialization-of-anonymous-classes/
2022-01-31 13:20:00.194+0000 [id=84] WARNING h.i.i.InstallUncaughtExceptionHandler#handleException: Caught unhandled exception with ID d7135add-6c31-4f76-8331-e8d609127f0b
2022-01-31 13:20:14.451+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 5 sec
2022-01-31 13:20:19.451+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 10 sec
2022-01-31 13:20:24.452+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 15 sec
2022-01-31 13:20:29.452+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 20 sec
2022-01-31 13:20:34.453+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 25 sec
2022-01-31 13:20:39.453+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 30 sec
2022-01-31 13:20:44.453+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 35 sec
2022-01-31 13:20:49.454+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 40 sec
2022-01-31 13:20:54.454+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 45 sec
2022-01-31 13:20:59.454+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 50 sec
2022-01-31 13:21:04.455+0000 [id=259] INFO o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#22]: checking /home/jenkins/jenkins/workspace/project-Build-pipe_3/project on linux unresponsive for 55 sec
2022-01-31 13:21:08.882+0000 [id=22] WARNING o.j.p.w.log.FileLogStorage#maybeFlush: failed to flush /home/mrigo/.jenkins/jobs/project-Build-pipe/builds/1526/log
java.io.IOException: Stream Closed
at java.io.FileOutputStream.writeBytes(Native Method)
at java.io.FileOutputStream.write(FileOutputStream.java:326)
at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$FlushControlledOutputStream.write(DelayBufferedOutputStream.java:127)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at org.jenkinsci.plugins.workflow.log.FileLogStorage.maybeFlush(FileLogStorage.java:193)
at org.jenkinsci.plugins.workflow.log.FileLogStorage.overallLog(FileLogStorage.java:202)
at org.jenkinsci.plugins.workflow.job.WorkflowRun.getLogText(WorkflowRun.java:1065)
at java.lang.invoke.MethodHandle.invokeWithArguments(MethodHandle.java:627)
at org.kohsuke.stapler.Function$MethodFunction.invoke(Function.java:393)
at org.kohsuke.stapler.Function$InstanceFunction.invoke(Function.java:405)
at org.kohsuke.stapler.MetaClass$2.doDispatch(MetaClass.java:220)
at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58)
at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898)
at org.kohsuke.stapler.MetaClass$9.dispatch(MetaClass.java:457)
at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898)
at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:281)
at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58)
at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898)
at org.kohsuke.stapler.Stapler.invoke(Stapler.java:694)
at org.kohsuke.stapler.Stapler.service(Stapler.java:240)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799)
at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1626)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:156)
at jenkins.telemetry.impl.UserLanguages$AcceptLanguageFilter.doFilter(UserLanguages.java:128)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153)
at jenkins.security.ResourceDomainFilter.doFilter(ResourceDomainFilter.java:80)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153)
at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:239)
at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:215)
at net.bull.javamelody.PluginMonitoringFilter.doFilter(PluginMonitoringFilter.java:88)
at org.jvnet.hudson.plugins.monitoring.HudsonMonitoringFilter.doFilter(HudsonMonitoringFilter.java:121)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153)
at jenkins.metrics.impl.MetricsFilter.doFilter(MetricsFilter.java:125)
at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:153)
at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:159)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:153)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:92)
at jenkins.security.AcegiSecurityExceptionFilter.doFilter(AcegiSecurityExceptionFilter.java:52)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:53)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:121)
at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:115)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:105)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:101)
at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:92)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:218)
at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:212)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.java:97)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:110)
at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:80)
at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:62)
at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:97)
at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:109)
at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:171)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:51)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:85)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at jenkins.security.SuspiciousRequestFilter.doFilter(SuspiciousRequestFilter.java:39)
at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:516)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:388)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:633)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:380)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:386)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
at java.lang. Thread .run( Thread .java:748)
The reason I was using the pipeline tee was not to mask my make's error output.
tee ( 'build.log' ){
sh 'make -j $(nproc --all)'
}
After modifying to this the make's return value still gets passed to the pipe, hence the pipe fails if make fails:
sh """#!/bin/bash
set -o pipefail
make -j \$(nproc --all) 2>&1 | tee build.log
"""
Maybe some of you out there also do use tee in pipelines.
Only thing I am wondering is how to do now tee in the windows builds that we do have, as the pipeline tee was actually quiet handy.
Christopher Head
added a comment - I don’t use the tee step anywhere, so it’s not exclusive to that.
Christopher Head
added a comment - I don’t use the tee step anywhere, so it’s not exclusive to that. 
Vishal kumar
added a comment - Any updates on this issue please ? What might be causing this ? what plugin combination might be causing this ?
Vishal kumar
added a comment - Any updates on this issue please ? What might be causing this ? what plugin combination might be causing this ? Mate Rigo
added a comment - The only feedback I can give is, that ever since I have removed all the jenkins pipeline tee commands, this issue has never reappeared.
I know I was told that it is not totally related to this, but yet this is some input that might interest folks who follow this thread.
Cheers!
Mate Rigo
added a comment - The only feedback I can give is, that ever since I have removed all the jenkins pipeline tee commands, this issue has never reappeared.
I know I was told that it is not totally related to this, but yet this is some input that might interest folks who follow this thread.
Cheers! 
Shambhu Sharma
added a comment - I am facing the same issue with latest Jenkins 2.361.1. Console log stops and getting "org.jenkinsci.plugins.workflow.log.FileLogStorage maybeFlush" in Jenkins log when Job with many parallel stages is trigger using Jira trigger Plugin. The same Job if executed manually in Jenkins, it is working fine.
Shambhu Sharma
added a comment - I am facing the same issue with latest Jenkins 2.361.1. Console log stops and getting "org.jenkinsci.plugins.workflow.log.FileLogStorage maybeFlush" in Jenkins log when Job with many parallel stages is trigger using Jira trigger Plugin. The same Job if executed manually in Jenkins, it is working fine. shambhu_1980 what kind of storage are you using? Local disk, network storage, etc.? What filesystem? Please share any mount options you are using for the Jenkins filesystem.
owenmehegan I am using automount NAS storage. On the same network storage, I have Jenkins 2.164.3 running on a different server and the same Job is running fine for more than a year. I am creating a new Jenkins server with latest version and facing the issue. Before the maybeflush log, I could see that durableTask plugin reporting unresponsive nodes on which different stages are running and then maybeFlush error shows and console log stops, but the job is still running in background. Just to add that Jenkins 2.361.1 is running with Java "11.0.16".
Shambhu Sharma
added a comment - - edited owenmehegan I am using automount NAS storage. On the same network storage, I have Jenkins 2.164.3 running on a different server and the same Job is running fine for more than a year. I am creating a new Jenkins server with latest version and facing the issue. Before the maybeflush log, I could see that durableTask plugin reporting unresponsive nodes on which different stages are running and then maybeFlush error shows and console log stops, but the job is still running in background. Just to add that Jenkins 2.361.1 is running with Java "11.0.16". Can you run 'mount' and share the line which shows your Jenkins volume, so I can see the mount options? We think this issue is triggered by filesystem performance.
Vishal kumar
added a comment - We are facing the same issue . Our Jenkins_home is PVC mounted supported by trident netapp storage. Jenkins is running on 2.319.1.
Latency on storage size file I/O ops is around 15 ms only .
root@nbu-sws-fw-57fd565887-5hhkl:/# df -k /var/jenkins
Filesystem 1K-blocks Used Available Use% Mounted on
XX.X.XX.X:/trident_pvc_XXXXXXXXXXXXXXXXXXXXXXXXX 115343360 32725952 82617408 29% /var/jenkins
Vishal kumar
added a comment - We are facing the same issue . Our Jenkins_home is PVC mounted supported by trident netapp storage. Jenkins is running on 2.319.1.
Latency on storage size file I/O ops is around 15 ms only .
root@nbu-sws-fw-57fd565887-5hhkl:/# df -k /var/jenkins
Filesystem 1K-blocks Used Available Use% Mounted on
XX.X.XX.X:/trident_pvc_XXXXXXXXXXXXXXXXXXXXXXXXX 115343360 32725952 82617408 29% /var/jenkins
I would love to see some NFS mount options... Run 'mount' and share output please.
Vishal kumar
added a comment - root@nyt-bhuW-fw-57fd565887-XXXXX:/var/jenkins# mount |grep nfs
XX.X.XXX.XXX:/trident_pvc_XXXXXXX_e948_XXXX_XXXX_XXXXXXXX on /var/jenkins type nfs (rw,relatime,vers=3,rsize=65536,wsize=65536,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=XX.X.XXX.XXX,mountvers=3,mountport=635,mountproto=tcp,local_lock=none,addr=XX.X.XXX.XXX)
XX.X.XXX.XXX:/trident_pvc_XXXXXX_624b_XXXX_XXXXX_XXXXXXXXX on /var/jenkins_backup type nfs (rw,relatime,vers=3,rsize=65536,wsize=65536,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=XX.X.XXX.XXX,mountvers=3,mountport=635,mountproto=tcp,local_lock=none,addr=XX.X.XXX.XXX)
Multiple other mounts on this host of type - cgroup , overlay , proc ,tmpfs ,devpts,sysfs,mqueue
Vishal kumar
added a comment - root@nyt-bhuW-fw-57fd565887-XXXXX:/var/jenkins# mount |grep nfs
XX.X.XXX.XXX:/trident_pvc_XXXXXXX_e948_XXXX_XXXX_XXXXXXXX on /var/jenkins type nfs (rw,relatime,vers=3,rsize=65536,wsize=65536,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=XX.X.XXX.XXX,mountvers=3,mountport=635,mountproto=tcp,local_lock=none,addr=XX.X.XXX.XXX)
XX.X.XXX.XXX:/trident_pvc_XXXXXX_624b_XXXX_XXXXX_XXXXXXXXX on /var/jenkins_backup type nfs (rw,relatime,vers=3,rsize=65536,wsize=65536,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=XX.X.XXX.XXX,mountvers=3,mountport=635,mountproto=tcp,local_lock=none,addr=XX.X.XXX.XXX)
Multiple other mounts on this host of type - cgroup , overlay , proc ,tmpfs ,devpts,sysfs,mqueue Shambhu Sharma
added a comment - owenmehegan Here is output of mount:
xxxxnasxx:/xxxx/xxxxnasxx/xxx/xxx on /yyy/xxx type nfs (rw,relatime,vers=3,rsize=131072,wsize=524288,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=xx.xxx.xxx.xx,mountvers=3,mountport=300,mountproto=tcp,local_lock=none,addr=xx.xxx.xxx.xx)
Shambhu Sharma
added a comment - owenmehegan Here is output of mount:
xxxxnasxx:/xxxx/xxxxnasxx/xxx/xxx on /yyy/xxx type nfs (rw,relatime,vers=3,rsize=131072,wsize=524288,namlen=255,hard,proto=tcp,timeo=600,retrans=2,sec=sys,mountaddr=xx.xxx.xxx.xx,mountvers=3,mountport=300,mountproto=tcp,local_lock=none,addr=xx.xxx.xxx.xx) CloudBees has recommended best practices for NFS performance tuning for the Jenkins workload, you can find those here: https://docs.cloudbees.com/docs/cloudbees-ci-kb/latest/client-and-managed-masters/nfs-guide This includes recommended mount options that we've found give the best results. vishakumar and shambhu_1980 in both of your cases I would recommend that you change the rsize and wsize values to 32768, remove 'relatime', and add 'noatime,nodiratime,async'. The smaller read and write block sizes should be a better fit because Jenkins performs a high volume of small read and write ops, different from some more typical storage workloads, so tuning this tends to help speed things up there.
The other thing I would suggest is that you look at the `dmesg` log after seeing this issue, to see if the kernel is logging any warnings about the storage.
shambhu_1980 you also mentioned that you have another server running Jenkins 2.164.3 and it doesn't have this issue - in later versions of Jenkins the logging is increased substantially, so that is probably why you don't see the problem on older versions. But again we think this is fundamentally a storage performance issue - CloudBees Support has seen this issue with just four customers, and all of them had a combination of very busy/intensive logging from jobs on their controllers, coupled with network storage for the JENKINS_HOME filesystem.
If someone could provide a sample Pipeline job which reliably reproduces this issue, an engineer could investigate and see if there is any bug fix or performance improvement that can be made. But without that, the best I can recommend is making the NFS tuning adjustments that I suggested, and also looking at your jobs to see if you can reduce the amount of log output they generate.
Shambhu Sharma
added a comment - owenmehegan Thanks for the pointers. I moved JENKINS_HOME to local disk and it is working fine. I would check with my storage team if they can make suggested change. The output log is from our build system (GNU make) and pipeline runs parallel instances of builds for different platforms, hence nothing much can be done to reduce the log output.
Shambhu Sharma
added a comment - owenmehegan Thanks for the pointers. I moved JENKINS_HOME to local disk and it is working fine. I would check with my storage team if they can make suggested change. The output log is from our build system (GNU make) and pipeline runs parallel instances of builds for different platforms, hence nothing much can be done to reduce the log output. Hi owenmehegan ,
I have also hit this issue recently and here is how I can reproduce it with following sample job:
MAX_STAGES=10 // Don't overload Jenkins NODE='linux' def test_stage(node_label, stage_name) { return { node(node_label) { stage(stage_name) { script { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { println("Total lines: $NUM_LOGS") sh '''#!/bin/bash set +x END=$((${NUM_LOGS})) for i in $(seq 1 $END); do echo "#$i: ping!"; done ''' } } } } } } pipeline { agent none parameters { string(name: 'NUM_LOGS', defaultValue: "10000", description: 'The number of log lines.', trim:true) string(name: 'NUM_STAGES', defaultValue: "3", description: "The number of parallel stages. Max ($MAX_STAGES)", trim:true) } options { buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '5', numToKeepStr: '')) timeout(time: 90, unit: 'MINUTES') quietPeriod(2) ansiColor('xterm') // Do not allow the pipeline to resume if the controller restarts disableResume() durabilityHint 'PERFORMANCE_OPTIMIZED' } stages { stage('RunTests') { steps { script { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { println(params.NUM_STAGES) def total_stages = "$NUM_STAGES" as Integer if (total_stages > MAX_STAGES) { error("Use stages less than $MAX_STAGES")} def stages = [:] for (i = 0; i < total_stages; i++) { stages["stage${i}"] = test_stage(NODE, "stage${i}") } parallel(stages) } } } } } }
Job parameters:
- NUM_LOGS = 50000
- NUM_STAGES = 3
When executed the job finishes before all the three stages finish displaying "ping" upto specified number (NUM_LOGS).
Sep 29, 2022 10:56:14 AM WARNING org.jenkinsci.plugins.workflow.log.FileLogStorage maybeFlush failed to flush /var/lib/jenkins/jobs/Test_JenkinsLog/builds/48/log java.io.IOException: Stream Closed at java.base/java.io.FileOutputStream.writeBytes(Native Method) at java.base/java.io.FileOutputStream.write(FileOutputStream.java:354) at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$FlushControlledOutputStream.write(DelayBufferedOutputStream.java:127) at java.base/java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) at java.base/java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) at java.base/java.io.FilterOutputStream.flush(FilterOutputStream.java:153) at org.jenkinsci.plugins.workflow.log.FileLogStorage.maybeFlush(FileLogStorage.java:193) at org.jenkinsci.plugins.workflow.log.FileLogStorage.overallLog(FileLogStorage.java:202) at org.jenkinsci.plugins.workflow.job.WorkflowRun.getLogText(WorkflowRun.java:1070) at java.base/java.lang.invoke.MethodHandle.invokeWithArguments(MethodHandle.java:710) at org.kohsuke.stapler.Function$MethodFunction.invoke(Function.java:397) at org.kohsuke.stapler.Function$InstanceFunction.invoke(Function.java:409) at org.kohsuke.stapler.MetaClass$2.doDispatch(MetaClass.java:224) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:59) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.MetaClass$9.dispatch(MetaClass.java:475) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:289) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:59) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:766) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:898) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:694) at org.kohsuke.stapler.Stapler.service(Stapler.java:240) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799) at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1631) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:157) at org.jenkinsci.plugins.ssegateway.Endpoint$SSEListenChannelFilter.doFilter(Endpoint.java:248) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154) at jenkins.security.ResourceDomainFilter.doFilter(ResourceDomainFilter.java:81) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154) at jenkins.telemetry.impl.UserLanguages$AcceptLanguageFilter.doFilter(UserLanguages.java:129) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154) at io.jenkins.blueocean.ResourceCacheControl.doFilter(ResourceCacheControl.java:134) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154) at io.jenkins.blueocean.auth.jwt.impl.JwtAuthenticationFilter.doFilter(JwtAuthenticationFilter.java:60) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154) at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:239) at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:215) at net.bull.javamelody.PluginMonitoringFilter.doFilter(PluginMonitoringFilter.java:88) at org.jvnet.hudson.plugins.monitoring.HudsonMonitoringFilter.doFilter(HudsonMonitoringFilter.java:121) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154) at jenkins.metrics.impl.MetricsFilter.doFilter(MetricsFilter.java:125) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154) at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:160) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:154) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:94) at jenkins.security.AcegiSecurityExceptionFilter.doFilter(AcegiSecurityExceptionFilter.java:52) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99) at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:54) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99) at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:122) at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:116) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99) at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:109) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99) at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:102) at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:93) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99) at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:219) at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:213) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99) at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.java:97) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:110) at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:80) at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:63) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:99) at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:111) at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:172) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:53) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:86) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at jenkins.security.SuspiciousRequestFilter.doFilter(SuspiciousRequestFilter.java:38) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:571) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1440) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1355) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:516) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base/java.lang.Thread.run(Thread.java:829)
We are using Jenkins v2.346.2.
xZanon none
added a comment - Hello,
We have upgraded to Jenkins Jenkins 2.361.1 , same issue .
maybe the key is in the NFS settings .
xZanon none
added a comment - Hello,
We have upgraded to Jenkins Jenkins 2.361.1 , same issue .
maybe the key is in the NFS settings .
irfank thanks for sharing this, I have passed it on in hopes that the issue can be reproduced elsewhere and fixed.
Was able to reproduce using the pipeline from irfank. Tested in 2 similar controllers, just storage is different: one over an NFS mount when the NFS Server storage is backed by a Google PD storage. One over an attached Google PD directly. The results:
- NFS: if I run this pipeline with NUM_LOGS at 1 000 000 and NUM_STAGES at 10, I reproduce the problem. The build takes 8/9 minutes to eventually complete.
- Not NFS: if I run the build with NUM_LOGS at 1 000 000 and NUM_STAGES at 10, I cannot reproduce the problem. The build takes ~3 min.
In both cases, I see occurrences of checking $WORKSPACE on $AGENT_NAME unresponsive for 5 sec, but we only see the problem with truncated logs when over NFS…
Note also that:
- NFS: if I run this pipeline with NUM_LOGS at 10 000 000 and NUM_STAGES at 1, *I cannot reproduce the problem*.
So NFS / slower storage is a factor. But the main one is parallel executions.
****
Attached 2 thread dumps taking at about ~10s interval while a build with 10 parallel seemed to not be writing logs anymore:
Allan BURDAJEWICZ
added a comment - - edited Was able to reproduce using the pipeline from irfank . Tested in 2 similar controllers, just storage is different: one over an NFS mount when the NFS Server storage is backed by a Google PD storage. One over an attached Google PD directly. The results:
NFS: if I run this pipeline with NUM_LOGS at 1 000 000 and NUM_STAGES at 10 , I reproduce the problem. The build takes 8/9 minutes to eventually complete.
Not NFS: if I run the build with NUM_LOGS at 1 000 000 and NUM_STAGES at 10 , I cannot reproduce the problem. The build takes ~3 min.
In both cases, I see occurrences of checking $WORKSPACE on $AGENT_NAME unresponsive for 5 sec , but we only see the problem with truncated logs when over NFS…
Note also that:
NFS: if I run this pipeline with NUM_LOGS at 10 000 000 and NUM_STAGES at 1, * I cannot reproduce the problem *.
So NFS / slower storage is a factor. But the main one is parallel executions.
****
Attached 2 thread dumps taking at about ~10s interval while a build with 10 parallel seemed to not be writing logs anymore:
thread-1.txt
thread-2.txt
Allan BURDAJEWICZ
added a comment - jglick dnusbaum I wonder what is your thought about this and also if maybe some tuning of the org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream could maybe help here ?
Allan BURDAJEWICZ
added a comment - jglick dnusbaum I wonder what is your thought about this and also if maybe some tuning of the org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream could maybe help here ? From a brief glance, I do not think “tuning” would suffice. Something is calling FileOutputStream.close on the build log before the build has completed, so that culprit needs to be identified and fixed. If the error is reproducible then I suppose it would suffice to patch FileLogStorage to wrap os in a FilterOutputStream which records a stack trace when close is called.
Allan BURDAJEWICZ
added a comment - I did use a FOS that print the thread dump on close(), see https://github.com/jenkinsci/workflow-api-plugin/compare/master...Dohbedoh:workflow-api-plugin:JENKINS-56446, and there is no call to close() while the issue happens. Eventually when the build completes, we see the call as expected:
java.lang.Throwable: JENKINS-56446 at org.jenkinsci.plugins.workflow.log.FileLogStorage$Jenkins56446FilterOutputStream.close(FileLogStorage.java:353) at java.base/java.io.FilterOutputStream.close(FilterOutputStream.java:188) at java.base/java.io.FilterOutputStream.close(FilterOutputStream.java:191) at java.base/java.io.FilterOutputStream.close(FilterOutputStream.java:191) at org.jenkinsci.plugins.workflow.log.FileLogStorage$IndexOutputStream.close(FileLogStorage.java:181) at java.base/java.io.PrintStream.close(PrintStream.java:439) at org.jenkinsci.plugins.workflow.log.BufferedBuildListener.close(BufferedBuildListener.java:60) at org.jenkinsci.plugins.workflow.log.TaskListenerDecorator$CloseableTaskListener.close(TaskListenerDecorator.java:300) at org.jenkinsci.plugins.workflow.job.WorkflowRun.finish(WorkflowRun.java:649) at org.jenkinsci.plugins.workflow.job.WorkflowRun$GraphL.onNewHead(WorkflowRun.java:1065) at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution.notifyListeners(CpsFlowExecution.java:1587) at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$3.run(CpsThreadGroup.java:509) at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$1.run(CpsVmExecutorService.java:38) at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:139) at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28) at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68) at jenkins.util.ErrorLoggingExecutorService.lambda$wrap$0(ErrorLoggingExecutorService.java:51) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829)
Allan BURDAJEWICZ
added a comment - I did use a FOS that print the thread dump on close() , see https://github.com/jenkinsci/workflow-api-plugin/compare/master...Dohbedoh:workflow-api-plugin:JENKINS-56446 , and there is no call to close() while the issue happens. Eventually when the build completes, we see the call as expected:
java.lang.Throwable: JENKINS-56446
at org.jenkinsci.plugins.workflow.log.FileLogStorage$Jenkins56446FilterOutputStream.close(FileLogStorage.java:353)
at java.base/java.io.FilterOutputStream.close(FilterOutputStream.java:188)
at java.base/java.io.FilterOutputStream.close(FilterOutputStream.java:191)
at java.base/java.io.FilterOutputStream.close(FilterOutputStream.java:191)
at org.jenkinsci.plugins.workflow.log.FileLogStorage$IndexOutputStream.close(FileLogStorage.java:181)
at java.base/java.io.PrintStream.close(PrintStream.java:439)
at org.jenkinsci.plugins.workflow.log.BufferedBuildListener.close(BufferedBuildListener.java:60)
at org.jenkinsci.plugins.workflow.log.TaskListenerDecorator$CloseableTaskListener.close(TaskListenerDecorator.java:300)
at org.jenkinsci.plugins.workflow.job.WorkflowRun.finish(WorkflowRun.java:649)
at org.jenkinsci.plugins.workflow.job.WorkflowRun$GraphL.onNewHead(WorkflowRun.java:1065)
at org.jenkinsci.plugins.workflow.cps.CpsFlowExecution.notifyListeners(CpsFlowExecution.java:1587)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$3.run(CpsThreadGroup.java:509)
at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$1.run(CpsVmExecutorService.java:38)
at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:139)
at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68)
at jenkins.util.ErrorLoggingExecutorService.lambda$wrap$0(ErrorLoggingExecutorService.java:51)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang. Thread .run( Thread .java:829)
Allan BURDAJEWICZ
added a comment - What I see when the log are not appended is that the os.getChannel().isOpen() is false and os.getFD().valid() also is false. Just I can't seem to capture the source of the closure... Will test further.
Allan BURDAJEWICZ
added a comment - What I see when the log are not appended is that the os.getChannel().isOpen() is false and os.getFD().valid() also is false . Just I can't seem to capture the source of the closure... Will test further. Hello,
I'm also affected by this issue.
To try to troubleshoot it, I created a small reproducer with this pipeline:
pipeline {
agent any
stages {
stage('Init'){
steps{
deleteDir()
}
}
stage('Hello') {
matrix {
axes {
axis {
name 'blarg'
values 'foo', 'bar', 'baz', 'qux', 'wizz'
}
}
stages {
stage('Main') {
steps {
sh """
for i in \$(seq 1 ${TURNS}); do
echo hello world \${i}!
done
"""
}
}
}
}
}
}
}
It requires one parameter, TURNS, which controls how many times the log message will be displayed.
To reproduce the issue in a controlled environment, I spawned a jenkins instance in a docker container and only installed the pipeline plugin. The storage is a volume pointing to a local SSD drive mounted in ext4.
I launched it with TURNS == 10 000 000 and jenkins started to misbehave around 2 493 402, but the numbers seem dependent on:
- the number of plugins installed
- the load (probably related to I/0)
The symptoms are:
- logs being truncated
- job taking ages to finish
- no possibility to kill the job properly, be it with cancel or using the /kill URL -> have to restart jenkins in the end.
When I click on the job in jenkin, I have, in the console, a stack trace like that which shows up:
2023-06-26 12:57:50.139+0000 [id=359] WARNING o.j.p.w.log.FileLogStorage#maybeFlush: failed to flush /var/jenkins_home/jobs/reproducer/builds/3/log
java.io.IOException: Stream Closed
at java.base/java.io.FileOutputStream.writeBytes(Native Method)
at java.base/java.io.FileOutputStream.write(FileOutputStream.java:354)
[...]
and frequent (~1Hz) messages like that:
o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#71]: checking /var/jenkins_home/workspace/reproducer on unresponsive for 33 min
Don't hesitate to tell if I can provide more valuable information.
[Edit] The jenkins version is 2.401.1, the latest docker image's version ATTOW.
Nicolas
added a comment - - edited Hello,
I'm also affected by this issue.
To try to troubleshoot it, I created a small reproducer with this pipeline:
pipeline {
agent any
stages {
stage( 'Init' ){
steps{
deleteDir()
}
}
stage( 'Hello' ) {
matrix {
axes {
axis {
name 'blarg'
values 'foo' , 'bar' , 'baz' , 'qux' , 'wizz'
}
}
stages {
stage( 'Main' ) {
steps {
sh """
for i in \$(seq 1 ${TURNS}); do
echo hello world \${i}!
done
"""
}
}
}
}
}
}
}
It requires one parameter, TURNS, which controls how many times the log message will be displayed.
To reproduce the issue in a controlled environment, I spawned a jenkins instance in a docker container and only installed the pipeline plugin. The storage is a volume pointing to a local SSD drive mounted in ext4.
I launched it with TURNS == 10 000 000 and jenkins started to misbehave around 2 493 402, but the numbers seem dependent on:
the number of plugins installed
the load (probably related to I/0)
The symptoms are:
logs being truncated
job taking ages to finish
no possibility to kill the job properly, be it with cancel or using the /kill URL -> have to restart jenkins in the end.
When I click on the job in jenkin, I have, in the console, a stack trace like that which shows up:
2023-06-26 12:57:50.139+0000 [id=359] WARNING o.j.p.w.log.FileLogStorage#maybeFlush: failed to flush / var /jenkins_home/jobs/reproducer/builds/3/log
java.io.IOException: Stream Closed
at java.base/java.io.FileOutputStream.writeBytes(Native Method)
at java.base/java.io.FileOutputStream.write(FileOutputStream.java:354)
[...]
and frequent (~1Hz) messages like that:
o.j.p.w.s.concurrent.Timeout#lambda$ping$0: org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#71]: checking / var /jenkins_home/workspace/reproducer on unresponsive for 33 min
Don't hesitate to tell if I can provide more valuable information.
[Edit] The jenkins version is 2.401.1, the latest docker image's version ATTOW. Nicolas
added a comment - I just tested with a Matrix job instead of a pipeline one and jenkins is able to cope with 10 000 000 (x5 axis) log lines without an issue.
So it looks like the issue is really in the pipeline plugin, or one of its dependencies.
Nicolas
added a comment - I just tested with a Matrix job instead of a pipeline one and jenkins is able to cope with 10 000 000 (x5 axis) log lines without an issue.
So it looks like the issue is really in the pipeline plugin, or one of its dependencies. https://github.com/openjdk/jdk/blob/63f32fbe9771b8200f707ed5d1d0e6555ad90f8b/src/java.base/share/native/libjava/io_util.c#L104-L106 suggests that there could be various OS-level problems causing this state, rather than an actual premature call to FileOutputStream.close.
maybeFlush already catches and logs the error, so I suspect this is just a miscellaneous symptom of the real problem.
Most of the reports come from people doing something apparently retrieves full build log text from inside the pipeline itself—though apparently not the most recent, from ncarrier, whose maybeFlush is I think triggered by the normal GUI build log display.
The unresponsive warnings are from org.jenkinsci.plugins.workflow.support.concurrent.Timeout as called from DurableTaskStep.Execution.check and generally indicate that there is excessive load on the agent, the controller, or the channel between the two which would prevent the sh step from reliably gathering new output and detecting when it should finish. Setting -Dorg.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep.USE_WATCHING=true (JENKINS-52165) may improve performance. Whether it would also cause this error to disappear, I cannot say.
I think I tracked down this issue. It happens if a thread is interrupted when writing to the log when a step transition happens. See https://github.com/jenkinsci/workflow-api-plugin/pull/296.
John Lengeling
added a comment - Thanks for tracking this down and fixing this. We have been running into the issue several times per month over the past year. Looking forward to testing and deploying this to our Jenkins servers.
John Lengeling
added a comment - Thanks for tracking this down and fixing this. We have been running into the issue several times per month over the past year. Looking forward to testing and deploying this to our Jenkins servers. A fix for this issue has been released in Pipeline: API plugin version 1239.vd7c497375cb_f.
Ran into this issue under Jenkins 2.176.3 with workflow-api 2.37, mailer 1.27 , email-ext 2.66. It occurred thou in the job post section trying to send an email notification. Over a couple of days, the stacktraces filled up the partition since it outputs about every 30s-60s.
Tried to abort the job but it would not die. Tried to kill the thread from Melody but it would not die. Had to restart Jenkins and the job started running again after the restart. Then I quickly killed it.
```
{{Sep 25, 2019 8:22:34 AM org.jenkinsci.plugins.workflow.log.FileLogStorage maybeFlush
WARNING: failed to flush /opt/apache/.jenkins/jobs/Reprogramming/jobs/verify/branches/master/builds/604/log
java.io.IOException: Stream Closed
at java.io.FileOutputStream.writeBytes(Native Method)
at java.io.FileOutputStream.write(FileOutputStream.java:326)
at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$FlushControlledOutputStream.write(DelayBufferedOutputStream.java:129)
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
at org.jenkinsci.plugins.workflow.log.FileLogStorage.maybeFlush(FileLogStorage.java:190)
at org.jenkinsci.plugins.workflow.log.FileLogStorage.overallLog(FileLogStorage.java:198)
at org.jenkinsci.plugins.workflow.job.WorkflowRun.getLogText(WorkflowRun.java:1019)
at hudson.plugins.emailext.AttachmentUtils$LogFileDataSource.getInputStream(AttachmentUtils.java:100)
at javax.mail.internet.MimeUtility.getEncoding(MimeUtility.java:197)
at javax.mail.internet.MimeUtility.getEncoding(MimeUtility.java:256)
at javax.mail.internet.MimeBodyPart.updateHeaders(MimeBodyPart.java:1366)
at javax.mail.internet.MimeBodyPart.updateHeaders(MimeBodyPart.java:1021)
at javax.mail.internet.MimeMultipart.updateHeaders(MimeMultipart.java:419)
at javax.mail.internet.MimeBodyPart.updateHeaders(MimeBodyPart.java:1345)
at javax.mail.internet.MimeMessage.updateHeaders(MimeMessage.java:2106)
at javax.mail.internet.MimeMessage.saveChanges(MimeMessage.java:2074)
at javax.mail.internet.MimeMessage.writeTo(MimeMessage.java:1769)
at com.sun.mail.smtp.SMTPTransport.sendMessage(SMTPTransport.java:1099)
at hudson.plugins.emailext.ExtendedEmailPublisher.sendMail(ExtendedEmailPublisher.java:484)
at hudson.plugins.emailext.EmailExtStep$EmailExtStepExecution.run(EmailExtStep.java:224)
at hudson.plugins.emailext.EmailExtStep$EmailExtStepExecution.run(EmailExtStep.java:163)
at org.jenkinsci.plugins.workflow.steps.AbstractSynchronousNonBlockingStepExecution$1$1.call(AbstractSynchronousNonBlockingStepExecution.java:47)
at hudson.security.ACL.impersonate(ACL.java:290)
at org.jenkinsci.plugins.workflow.steps.AbstractSynchronousNonBlockingStepExecution$1.run(AbstractSynchronousNonBlockingStepExecution.java:44)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)}}
```