-
Bug
-
Resolution: Unresolved
-
Critical
-
None
I had working setup of jenkins integrated with kubernetes, suddenly i started to get below from multicontainer jobs, specifically while using "Sh" in declarative pipeline
io.fabric8.kubernetes.client.http.WebSocketHandshakeException at io.fabric8.kubernetes.client.okhttp.OkHttpWebSocketImpl$BuilderImpl$1.onFailure(OkHttpWebSocketImpl.java:66) at okhttp3.internal.ws.RealWebSocket.failWebSocket(RealWebSocket.java:571) at okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:198) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203) at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) Suppressed: java.lang.Throwable: waiting here at io.fabric8.kubernetes.client.utils.Utils.waitUntilReady(Utils.java:168) at io.fabric8.kubernetes.client.utils.Utils.waitUntilReadyOrFail(Utils.java:179) at io.fabric8.kubernetes.client.dsl.internal.core.v1.PodOperationsImpl.exec(PodOperationsImpl.java:332) at io.fabric8.kubernetes.client.dsl.internal.core.v1.PodOperationsImpl.exec(PodOperationsImpl.java:85) at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.doLaunch(ContainerExecDecorator.java:428) at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.launch(ContainerExecDecorator.java:331) at hudson.Launcher$ProcStarter.start(Launcher.java:508) at org.jenkinsci.plugins.durabletask.BourneShellScript.launchWithCookie(BourneShellScript.java:176) at org.jenkinsci.plugins.durabletask.FileMonitoringTask.launch(FileMonitoringTask.java:132) at org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep$Execution.start(DurableTaskStep.java:324) at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:319) at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:193) at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:122) at jdk.internal.reflect.GeneratedMethodAccessor593.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.base/java.lang.reflect.Method.invoke(Unknown Source) at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93) at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1213) at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1022) at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:42) at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:48) at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:113) at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:163) at org.kohsuke.groovy.sandbox.GroovyInterceptor.onMethodCall(GroovyInterceptor.java:23) at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onMethodCall(SandboxInterceptor.java:158) at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:161) at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:165) at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135) at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.methodCall(SandboxInvoker.java:17) at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:86) at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:113) at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:83) at jdk.internal.reflect.GeneratedMethodAccessor232.invoke(Unknown Source) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) at java.base/java.lang.reflect.Method.invoke(Unknown Source) at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72) at com.cloudbees.groovy.cps.impl.ConstantBlock.eval(ConstantBlock.java:21) at com.cloudbees.groovy.cps.Next.step(Next.java:83) at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:174) at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:163) at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:129) at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:268) at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:163) at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$001(SandboxContinuable.java:18) at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:51) at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:185) at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:403) at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.access$400(CpsThreadGroup.java:97) at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:315) at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:279) at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:67) at java.base/java.util.concurrent.FutureTask.run(Unknown Source) at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:139) at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28) at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.base/java.util.concurrent.FutureTask.run(Unknown Source) ... 3 more Caused by: java.net.ProtocolException: Expected HTTP 101 response but was '500 Internal Server Error' at okhttp3.internal.ws.RealWebSocket.checkResponse(RealWebSocket.java:229) at okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:196) ... 5 more
Note: All plugins and jenkins is in latest version
[JENKINS-68234] Kubernetes pod failed with WebSocketHandshakeException
Uziel Sulkies
added a comment - We also experience this error.
I suspect it's connected with the other error we see:
io.fabric8.kubernetes.client.KubernetesClientException: not ready after 70000 MILLISECONDS (we increased to 70000 from 5000)
We configured the Max connections to kubernetes API to 10000 (although no clear way I know to make sure the configuration in the UI is the one used by the plugin).
We can't find any 500 response code on the Kubernetes API logs (on cloudwatch, for EKS), though.
Uziel Sulkies
added a comment - We also experience this error.
I suspect it's connected with the other error we see:
io.fabric8.kubernetes.client.KubernetesClientException: not ready after 70000 MILLISECONDS (we increased to 70000 from 5000)
We configured the Max connections to kubernetes API to 10000 (although no clear way I know to make sure the configuration in the UI is the one used by the plugin).
We can't find any 500 response code on the Kubernetes API logs (on cloudwatch, for EKS), though. Since an error 500 is reported, there should be some some error showing up in the kube-apiserver logs or maybe the kubelet logs of the node where the agent pod is running.
Do you have a simple reproduction scenario ? (simple pipeline script, pod yaml)
Allan BURDAJEWICZ
added a comment - - edited Since an error 500 is reported, there should be some some error showing up in the kube-apiserver logs or maybe the kubelet logs of the node where the agent pod is running.
Do you have a simple reproduction scenario ? (simple pipeline script, pod yaml) I have misconfigured defaultcontainer in agent block, Its resolved after using right container name from pod template.
Please raise appropriate error msg while using container name which is not available in pod template.
Francisco Aguiar
added a comment - nagavijayan
Did you solve this? Francisco Aguiar
made changes -
| Status | Original: Open [ 1 ] | New: In Progress [ 3 ] |
Francisco Aguiar
made changes -
| Status | Original: In Progress [ 3 ] | New: Open [ 1 ] |
Jeff Hughes
made changes -
| Comment |
[ I'm having this issue too.
GKE: v1.22. Jenkins: 2.359 Plugins: kubernetes:3663.v1c1e0ec5b_650 kubernetes-client-api:5.12.2-193.v26a_6078f65a_9 This started happening to us a few days ago when everything was working completely fine before. ] |
I am facing similar problem.
from the Log:
12:26:06 [Pipeline] { 12:26:06 [Pipeline] script 12:26:06 [Pipeline] { 12:26:06 [Pipeline] sh 12:26:21 io.fabric8.kubernetes.client.http.WebSocketHandshakeException 12:26:21 at io.fabric8.kubernetes.client.okhttp.OkHttpWebSocketImpl$BuilderImpl$1.onFailure(OkHttpWebSocketImpl.java:66) 12:26:21 at okhttp3.internal.ws.RealWebSocket.failWebSocket(RealWebSocket.java:571) 12:26:21 at okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:198) 12:26:21 at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203) 12:26:21 at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) 12:26:21 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) 12:26:21 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) 12:26:21 at java.lang.Thread.run(Thread.java:748) 12:26:21 Suppressed: java.lang.Throwable: waiting here 12:26:21 at io.fabric8.kubernetes.client.utils.Utils.waitUntilReady(Utils.java:168) 12:26:21 at io.fabric8.kubernetes.client.utils.Utils.waitUntilReadyOrFail(Utils.java:179) 12:26:21 at io.fabric8.kubernetes.client.dsl.internal.core.v1.PodOperationsImpl.exec(PodOperationsImpl.java:332) 12:26:21 at io.fabric8.kubernetes.client.dsl.internal.core.v1.PodOperationsImpl.exec(PodOperationsImpl.java:85) 12:26:21 at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.doLaunch(ContainerExecDecorator.java:428) 12:26:21 at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.launch(ContainerExecDecorator.java:331) 12:26:21 at hudson.Launcher$ProcStarter.start(Launcher.java:509) 12:26:21 at org.jenkinsci.plugins.durabletask.BourneShellScript.launchWithCookie(BourneShellScript.java:176) 12:26:21 at org.jenkinsci.plugins.durabletask.FileMonitoringTask.launch(FileMonitoringTask.java:132) 12:26:21 at org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep$Execution.start(DurableTaskStep.java:324) 12:26:21 at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:322) 12:26:21 at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:196) 12:26:21 at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:124) 12:26:21 at sun.reflect.GeneratedMethodAccessor235.invoke(Unknown Source) 12:26:21 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 12:26:21 at java.lang.reflect.Method.invoke(Method.java:498) 12:26:21 at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98) 12:26:21 at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325) 12:26:21 at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1225) 12:26:21 at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1034) 12:26:21 at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:41) 12:26:21 at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:47) 12:26:21 at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:116) 12:26:21 at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:163) 12:26:21 at org.kohsuke.groovy.sandbox.GroovyInterceptor.onMethodCall(GroovyInterceptor.java:23) 12:26:21 at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onMethodCall(SandboxInterceptor.java:158) 12:26:21 at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:161) 12:26:21 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:165) 12:26:21 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135) 12:26:21 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135) 12:26:21 at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.methodCall(SandboxInvoker.java:17) 12:26:21 at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:86) 12:26:21 at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:113) 12:26:21 at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:83) 12:26:21 at sun.reflect.GeneratedMethodAccessor172.invoke(Unknown Source) 12:26:21 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 12:26:21 at java.lang.reflect.Method.invoke(Method.java:498) 12:26:21 at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72) 12:26:21 at com.cloudbees.groovy.cps.impl.ConstantBlock.eval(ConstantBlock.java:21) 12:26:21 at com.cloudbees.groovy.cps.Next.step(Next.java:83) 12:26:21 at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:174) 12:26:21 at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:163) 12:26:21 at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:136) 12:26:21 at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:275) 12:26:21 at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:163) 12:26:21 at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$001(SandboxContinuable.java:18) 12:26:21 at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:51) 12:26:21 at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:187) 12:26:21 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:420) 12:26:21 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.access$400(CpsThreadGroup.java:95) 12:26:21 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:330) 12:26:21 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:294) 12:26:21 at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:67) 12:26:21 at java.util.concurrent.FutureTask.run(FutureTask.java:266) 12:26:21 at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:139) 12:26:21 at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28) 12:26:21 at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68) 12:26:21 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) 12:26:21 at java.util.concurrent.FutureTask.run(FutureTask.java:266) 12:26:21 ... 3 more 12:26:21 Caused by: java.net.ProtocolException: Expected HTTP 101 response but was '500 Internal Server Error' 12:26:21 at okhttp3.internal.ws.RealWebSocket.checkResponse(RealWebSocket.java:229) 12:26:21 at okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:196) 12:26:21 ... 5 more 12:26:21 [Pipeline] } 12:26:21 [Pipeline] // script 12:26:21 [Pipeline] }