Uploaded image for project: 'Jenkins'
  1. Jenkins
  2. JENKINS-55712

SocketTimeoutException with kubernetes pipelines

    XMLWordPrintable

Details

    Description

      We are facing issue with occasional SocketTimeoutException when executing long-running shell commands inside pod containers.

      We have never seen this happen during short running tasks.

      Full traceback:

      java.net.SocketException: Socket closed
      at java.net.SocketInputStream.read(SocketInputStream.java:204)
      at java.net.SocketInputStream.read(SocketInputStream.java:141)
      at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
      at sun.security.ssl.InputRecord.read(InputRecord.java:503)
      at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983)
      at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:940)
      at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
      at okio.Okio$2.read(Okio.java:139)
      at okio.AsyncTimeout$2.read(AsyncTimeout.java:237)
      Caused: java.net.SocketTimeoutException: timeout
      at okio.Okio$4.newTimeoutException(Okio.java:230)
      at okio.AsyncTimeout.exit(AsyncTimeout.java:285)
      at okio.AsyncTimeout$2.read(AsyncTimeout.java:241)
      at okio.RealBufferedSource.indexOf(RealBufferedSource.java:345)
      at okio.RealBufferedSource.readUtf8LineStrict(RealBufferedSource.java:217)
      at okhttp3.internal.http1.Http1Codec.readHeaderLine(Http1Codec.java:212)
      at okhttp3.internal.http1.Http1Codec.readResponseHeaders(Http1Codec.java:189)
      at okhttp3.internal.http.CallServerInterceptor.intercept(CallServerInterceptor.java:88)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
      at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:45)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
      at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
      at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
      at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:125)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
      at io.fabric8.kubernetes.client.utils.BackwardsCompatibilityInterceptor.intercept(BackwardsCompatibilityInterceptor.java:119)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
      at io.fabric8.kubernetes.client.utils.ImpersonatorInterceptor.intercept(ImpersonatorInterceptor.java:66)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
      at io.fabric8.kubernetes.client.utils.HttpClientUtils$2.intercept(HttpClientUtils.java:107)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
      at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
      at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:200)
      at okhttp3.RealCall.execute(RealCall.java:77)
      at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:379)
      at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:344)
      at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:313)
      at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:296)
      at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleGet(BaseOperation.java:794)
      at io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:210)
      at io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:177)
      at io.fabric8.kubernetes.client.dsl.base.BaseOperation.waitUntilExists(BaseOperation.java:967)
      at io.fabric8.kubernetes.client.dsl.base.HasMetadataOperation.waitUntilReady(HasMetadataOperation.java:223)
      at io.fabric8.kubernetes.client.dsl.base.HasMetadataOperation.waitUntilReady(HasMetadataOperation.java:37)
      at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.waitUntilContainerIsReady(ContainerExecDecorator.java:417)
      at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.doLaunch(ContainerExecDecorator.java:255)
      at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.launch(ContainerExecDecorator.java:236)
      at hudson.Launcher$ProcStarter.start(Launcher.java:455)
      at org.jenkinsci.plugins.durabletask.BourneShellScript.launchWithCookie(BourneShellScript.java:188)
      at org.jenkinsci.plugins.durabletask.FileMonitoringTask.launch(FileMonitoringTask.java:99)
      at org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep$Execution.start(DurableTaskStep.java:278)
      at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:270)
      at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:178)
      at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:122)
      at sun.reflect.GeneratedMethodAccessor15226.invoke(Unknown Source)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:498)
      at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93)
      at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325)
      at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1213)
      at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1022)
      at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:42)
      at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:48)
      at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:113)
      at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:157)
      at org.kohsuke.groovy.sandbox.GroovyInterceptor.onMethodCall(GroovyInterceptor.java:23)
      at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onMethodCall(SandboxInterceptor.java:155)
      at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:155)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:159)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.methodCall(SandboxInvoker.java:17)
      Caused: io.fabric8.kubernetes.client.KubernetesClientException: Operation: [get] for kind: [Pod] with name: [jenkins-slave-qlgv9-f2qlk] in namespace: [jenkins] failed.
      at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:64)
      at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:72)
      at io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:221)
      at io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:177)
      at io.fabric8.kubernetes.client.dsl.base.BaseOperation.waitUntilExists(BaseOperation.java:967)
      at io.fabric8.kubernetes.client.dsl.base.HasMetadataOperation.waitUntilReady(HasMetadataOperation.java:223)
      at io.fabric8.kubernetes.client.dsl.base.HasMetadataOperation.waitUntilReady(HasMetadataOperation.java:37)
      at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.waitUntilContainerIsReady(ContainerExecDecorator.java:417)
      at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.doLaunch(ContainerExecDecorator.java:255)
      at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.launch(ContainerExecDecorator.java:236)
      at hudson.Launcher$ProcStarter.start(Launcher.java:455)
      at org.jenkinsci.plugins.durabletask.BourneShellScript.launchWithCookie(BourneShellScript.java:188)
      at org.jenkinsci.plugins.durabletask.FileMonitoringTask.launch(FileMonitoringTask.java:99)
      at org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep$Execution.start(DurableTaskStep.java:278)
      at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:270)
      at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:178)
      at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:122)
      at sun.reflect.GeneratedMethodAccessor15226.invoke(Unknown Source)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:498)
      at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93)
      at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325)
      at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1213)
      at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1022)
      at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:42)
      at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:48)
      at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:113)
      at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:157)
      at org.kohsuke.groovy.sandbox.GroovyInterceptor.onMethodCall(GroovyInterceptor.java:23)
      at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onMethodCall(SandboxInterceptor.java:155)
      at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:155)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:159)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:129)
      at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.methodCall(SandboxInvoker.java:17)
      at WorkflowScript.run(WorkflowScript:63)
      at __cps.transform__(Native Method)
      at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:57)
      at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:109)
      at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:82)
      at sun.reflect.GeneratedMethodAccessor116.invoke(Unknown Source)
      at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      at java.lang.reflect.Method.invoke(Method.java:498)
      at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
      at com.cloudbees.groovy.cps.impl.ConstantBlock.eval(ConstantBlock.java:21)
      at com.cloudbees.groovy.cps.Next.step(Next.java:83)
      at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:174)
      at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:163)
      at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:129)
      at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:268)
      at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:163)
      at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$101(SandboxContinuable.java:34)
      at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.lambda$run0$0(SandboxContinuable.java:59)
      at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.GroovySandbox.runInSandbox(GroovySandbox.java:108)
      at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:58)
      at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:182)
      at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:332)
      at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.access$200(CpsThreadGroup.java:83)
      at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:244)
      at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:232)
      at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:64)
      at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:131)
      at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
      at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:59)
      at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
      at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
      at java.lang.Thread.run(Thread.java:748)
       

      Attachments

        Activity

          People

            iocanel Ioannis Canellos
            genunix Filip Pytloun
            Votes:
            2 Vote for this issue
            Watchers:
            3 Start watching this issue

            Dates

              Created:
              Updated: