-
Bug
-
Resolution: Unresolved
-
Minor
-
None
-
LTS 2.222.4
Remoting 4.2.1 (on both side)
ace-editor:1.1
allure-jenkins-plugin:2.28.1
analysis-model-api:8.0.1
ansicolor:0.6.3
ant:1.11
antisamy-markup-formatter:2.0
apache-httpcomponents-client-4-api:4.5.10-2.0
appcenter:0.9.0
atlassian-bitbucket-server-integration:1.1.0
authentication-tokens:1.3
authorize-project:1.3.0
aws-credentials:1.28
aws-java-sdk:1.11.723
azure-ad:1.2.0
azure-commons:1.0.4
azure-credentials:2.0.2
azure-keyvault:1.9
azure-vm-agents:1.4.0
badge:1.8
basic-branch-build-strategies:1.3.2
bootstrap4-api:4.4.1-10
bouncycastle-api:2.18
branch-api:2.5.5
build-monitor-plugin:1.12+build.201809061734
build-timeout:1.19.1
build-token-root:1.6
build-user-vars-plugin:1.5
chatter-notifier:2.1.1
cloud-stats:0.25
cloudbees-bitbucket-branch-source:2.7.0
cloudbees-folder:6.11.1
code-coverage-api:1.1.5
command-launcher:1.4
config-file-provider:3.6.3
configuration-as-code:1.36
copyartifact:1.43.1
credentials-binding:1.23
credentials:2.3.5
cron_column:1.4
crowd2:2.0.2
cucumber-reports:5.0.2
custom-tools-plugin:0.7
data-tables-api:1.10.20-14
dependency-check-jenkins-plugin:5.1.1
display-url-api:2.3.2
docker-commons:1.16
docker-workflow:1.23
durable-task:1.34
echarts-api:4.6.0-8
email-ext:2.69
extended-choice-parameter:0.78
external-monitor-job:1.7
extra-tool-installers:0.5
file-operations:1.9
font-awesome-api:5.12.0-7
forensics-api:0.7.0
git-client:3.2.1
git-server:1.9
git:4.2.2
github-api:1.106
github-branch-source:2.6.0
github:1.29.5
google-oauth-plugin:1.0.0
gradle:1.36
gravatar:2.1
handlebars:1.1.1
handy-uri-templates-2-api:2.1.8-1.0
hashicorp-vault-plugin:3.2.0
htmlpublisher:1.22
http_request:1.8.24
ivy:2.1
jackson2-api:2.10.3
javadoc:1.5
jdk-tool:1.4
job-dsl:1.77
jobConfigHistory:2.25
jquery-detached:1.2.1
jquery3-api:3.4.1-10
jquery:1.12.4-1
jsch:0.1.55.2
junit:1.28
kubernetes-cd:2.3.0
kubernetes-client-api:4.8.0-1
kubernetes-credentials:0.6.2
kubernetes:1.25.1
ldap:1.21
locale:1.4
lockable-resources:2.7
logfilesizechecker:1.5
login-theme:1.1
mailer:1.30
mapdb-api:1.0.9.0
matrix-auth:2.5
matrix-project:1.14
maven-plugin:3.5
mercurial:2.8
metrics:4.0.2.6
momentjs:1.1.1
next-executions:1.0.15
nodejs:1.3.4
oauth-credentials:0.4
pam-auth:1.6
pipeline-aws:1.41
pipeline-build-step:2.12
pipeline-github-lib:1.0
pipeline-graph-analysis:1.10
pipeline-input-step:2.11
pipeline-milestone-step:1.3.1
pipeline-model-api:1.6.0
pipeline-model-declarative-agent:1.1.1
pipeline-model-definition:1.6.0
pipeline-model-extensions:1.6.0
pipeline-rest-api:2.13
pipeline-stage-step:2.3
pipeline-stage-tags-metadata:1.6.0
pipeline-stage-view:2.13
pipeline-utility-steps:2.5.0
plain-credentials:1.7
plugin-util-api:1.0.2
popper-api:1.16.0-6
prometheus:2.0.6
promoted-builds:3.5
resource-disposer:0.14
scm-api:2.6.3
script-security:1.71
simple-theme-plugin:0.6
skip-notifications-trait:1.0.5
sonar:2.11
ssh-agent:1.19
ssh-credentials:1.18.1
ssh-slaves:1.31.1
structs:1.20
subversion:2.13.1
tap:2.3
timestamper:1.11.2
token-macro:2.12
trilead-api:1.0.6
variant:1.3
view-job-filters:2.1.1
violation-comments-to-stash:1.116
warnings-ng:8.1.0
windows-slaves:1.6
workflow-aggregator:2.6
workflow-api:2.40
workflow-basic-steps:2.19
workflow-cps-global-lib:2.16
workflow-cps:2.80
workflow-durable-task-step:2.35
workflow-job:2.38
workflow-multibranch:2.21
workflow-scm-step:2.10
workflow-step-api:2.22
workflow-support:3.4
ws-cleanup:0.38LTS 2.222.4 Remoting 4.2.1 (on both side) ace-editor:1.1 allure-jenkins-plugin:2.28.1 analysis-model-api:8.0.1 ansicolor:0.6.3 ant:1.11 antisamy-markup-formatter:2.0 apache-httpcomponents-client-4-api:4.5.10-2.0 appcenter:0.9.0 atlassian-bitbucket-server-integration:1.1.0 authentication-tokens:1.3 authorize-project:1.3.0 aws-credentials:1.28 aws-java-sdk:1.11.723 azure-ad:1.2.0 azure-commons:1.0.4 azure-credentials:2.0.2 azure-keyvault:1.9 azure-vm-agents:1.4.0 badge:1.8 basic-branch-build-strategies:1.3.2 bootstrap4-api:4.4.1-10 bouncycastle-api:2.18 branch-api:2.5.5 build-monitor-plugin:1.12+build.201809061734 build-timeout:1.19.1 build-token-root:1.6 build-user-vars-plugin:1.5 chatter-notifier:2.1.1 cloud-stats:0.25 cloudbees-bitbucket-branch-source:2.7.0 cloudbees-folder:6.11.1 code-coverage-api:1.1.5 command-launcher:1.4 config-file-provider:3.6.3 configuration-as-code:1.36 copyartifact:1.43.1 credentials-binding:1.23 credentials:2.3.5 cron_column:1.4 crowd2:2.0.2 cucumber-reports:5.0.2 custom-tools-plugin:0.7 data-tables-api:1.10.20-14 dependency-check-jenkins-plugin:5.1.1 display-url-api:2.3.2 docker-commons:1.16 docker-workflow:1.23 durable-task:1.34 echarts-api:4.6.0-8 email-ext:2.69 extended-choice-parameter:0.78 external-monitor-job:1.7 extra-tool-installers:0.5 file-operations:1.9 font-awesome-api:5.12.0-7 forensics-api:0.7.0 git-client:3.2.1 git-server:1.9 git:4.2.2 github-api:1.106 github-branch-source:2.6.0 github:1.29.5 google-oauth-plugin:1.0.0 gradle:1.36 gravatar:2.1 handlebars:1.1.1 handy-uri-templates-2-api:2.1.8-1.0 hashicorp-vault-plugin:3.2.0 htmlpublisher:1.22 http_request:1.8.24 ivy:2.1 jackson2-api:2.10.3 javadoc:1.5 jdk-tool:1.4 job-dsl:1.77 jobConfigHistory:2.25 jquery-detached:1.2.1 jquery3-api:3.4.1-10 jquery:1.12.4-1 jsch:0.1.55.2 junit:1.28 kubernetes-cd:2.3.0 kubernetes-client-api:4.8.0-1 kubernetes-credentials:0.6.2 kubernetes:1.25.1 ldap:1.21 locale:1.4 lockable-resources:2.7 logfilesizechecker:1.5 login-theme:1.1 mailer:1.30 mapdb-api:1.0.9.0 matrix-auth:2.5 matrix-project:1.14 maven-plugin:3.5 mercurial:2.8 metrics:4.0.2.6 momentjs:1.1.1 next-executions:1.0.15 nodejs:1.3.4 oauth-credentials:0.4 pam-auth:1.6 pipeline-aws:1.41 pipeline-build-step:2.12 pipeline-github-lib:1.0 pipeline-graph-analysis:1.10 pipeline-input-step:2.11 pipeline-milestone-step:1.3.1 pipeline-model-api:1.6.0 pipeline-model-declarative-agent:1.1.1 pipeline-model-definition:1.6.0 pipeline-model-extensions:1.6.0 pipeline-rest-api:2.13 pipeline-stage-step:2.3 pipeline-stage-tags-metadata:1.6.0 pipeline-stage-view:2.13 pipeline-utility-steps:2.5.0 plain-credentials:1.7 plugin-util-api:1.0.2 popper-api:1.16.0-6 prometheus:2.0.6 promoted-builds:3.5 resource-disposer:0.14 scm-api:2.6.3 script-security:1.71 simple-theme-plugin:0.6 skip-notifications-trait:1.0.5 sonar:2.11 ssh-agent:1.19 ssh-credentials:1.18.1 ssh-slaves:1.31.1 structs:1.20 subversion:2.13.1 tap:2.3 timestamper:1.11.2 token-macro:2.12 trilead-api:1.0.6 variant:1.3 view-job-filters:2.1.1 violation-comments-to-stash:1.116 warnings-ng:8.1.0 windows-slaves:1.6 workflow-aggregator:2.6 workflow-api:2.40 workflow-basic-steps:2.19 workflow-cps-global-lib:2.16 workflow-cps:2.80 workflow-durable-task-step:2.35 workflow-job:2.38 workflow-multibranch:2.21 workflow-scm-step:2.10 workflow-step-api:2.22 workflow-support:3.4 ws-cleanup:0.38
Hi,
Since we upgraded to Jenkins Core 2.222.4 (to include the fix JENKINS-61409) and remoting 4.2.1
We are facing much more stability issue on the websocket connection. It was not the case before with remoting 4.2 (The only issues we faced was the large payload).
We can observe now, disconnection on the middle of builds
Connection break after a simple git checkout.
[Pipeline] { (Git Checkout) [Pipeline] dir 10:08:31 Running in /home/jenkins/agent/workspace/workspace/***** [Pipeline] { [Pipeline] checkout (hide) [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } 10:08:45 ********* was marked offline: Connection was broken: java.nio.channels.ClosedChannelException 10:08:45 at jenkins.agents.WebSocketAgents$Session.closed(WebSocketAgents.java:141) 10:08:45 at jenkins.websocket.WebSocketSession.onWebSocketSomething(WebSocketSession.java:91) 10:08:45 at com.sun.proxy.$Proxy91.onWebSocketClose(Unknown Source) 10:08:45 at
On the agent (multiple exception)
Jun 05, 2020 8:07:10 AM org.jenkinsci.plugins.workflow.log.GCFlushedOutputStream$FlushRef lambda$static$0 WARNING: null hudson.remoting.ChannelClosedException: Channel "hudson.remoting.Channel@6cca5774:*****": channel is already closed at hudson.remoting.Channel.send(Channel.java:760) at hudson.remoting.ProxyOutputStream.flush(ProxyOutputStream.java:155) at hudson.remoting.RemoteOutputStream.flush(RemoteOutputStream.java:112) at java.io.FilterOutputStream.flush(FilterOutputStream.java:140) at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream$FlushControlledOutputStream.flush(DelayBufferedOutputStream.java:131) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:141) at org.jenkinsci.plugins.workflow.log.GCFlushedOutputStream$FlushRef.lambda$static$0(GCFlushedOutputStream.java:77) at jenkins.security.ImpersonatingScheduledExecutorService$1.run(ImpersonatingScheduledExecutorService.java:58) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: hudson.remoting.ChannelClosedException: Channel "hudson.remoting.Channel@6cca5774:******": channel is already closed at hudson.remoting.Engine$1AgentEndpoint.onClose(Engine.java:590) at io.jenkins.remoting.shaded.org.glassfish.tyrus.core.TyrusEndpointWrapper.onClose(TyrusEndpointWrapper.java:1251) at io.jenkins.remoting.shaded.org.glassfish.tyrus.core.TyrusWebSocket.onClose(TyrusWebSocket.java:130) at io.jenkins.remoting.shaded.org.glassfish.tyrus.core.ProtocolHandler.close(ProtocolHandler.java:469) at io.jenkins.remoting.shaded.org.glassfish.tyrus.core.TyrusWebSocket.close(TyrusWebSocket.java:260) at io.jenkins.remoting.shaded.org.glassfish.tyrus.client.TyrusClientEngine$2$1.close(TyrusClientEngine.java:635) at io.jenkins.remoting.shaded.org.glassfish.tyrus.container.jdk.client.ClientFilter.processError(ClientFilter.java:254) at io.jenkins.remoting.shaded.org.glassfish.tyrus.container.jdk.client.Filter.onError(Filter.java:180) at io.jenkins.remoting.shaded.org.glassfish.tyrus.container.jdk.client.Filter.onError(Filter.java:183) at io.jenkins.remoting.shaded.org.glassfish.tyrus.container.jdk.client.Filter.onError(Filter.java:183) at io.jenkins.remoting.shaded.org.glassfish.tyrus.container.jdk.client.Filter.onError(Filter.java:183) at io.jenkins.remoting.shaded.org.glassfish.tyrus.container.jdk.client.TransportFilter$4.failed(TransportFilter.java:314) at io.jenkins.remoting.shaded.org.glassfish.tyrus.container.jdk.client.TransportFilter$4.failed(TransportFilter.java:283) at sun.nio.ch.Invoker.invokeUnchecked(Invoker.java:128) at sun.nio.ch.Invoker$2.run(Invoker.java:218) at sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:112) ... 3 more Jun 05, 2020 8:07:28 AM io.jenkins.remoting.shaded.org.glassfish.tyrus.container.jdk.client.ClientFilter processError SEVERE: Connection error has occurred java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.UnixAsynchronousSocketChannelImpl.finishRead(UnixAsynchronousSocketChannelImpl.java:388) at sun.nio.ch.UnixAsynchronousSocketChannelImpl.finish(UnixAsynchronousSocketChannelImpl.java:191) at sun.nio.ch.UnixAsynchronousSocketChannelImpl.onEvent(UnixAsynchronousSocketChannelImpl.java:213) at sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:293) at java.lang.Thread.run(Thread.java:748) WARNING: LinkageError while performing UserRequest:hudson.FilePath$IsDirectory@4d6ab49 java.lang.NoClassDefFoundError: hudson/util/io/Archiver at java.lang.Class.getDeclaredFields0(Native Method) at java.lang.Class.privateGetDeclaredFields(Class.java:2583) at java.lang.Class.getDeclaredFields(Class.java:1916) at java.io.ObjectStreamClass.getDefaultSerialFields(ObjectStreamClass.java:1851) at java.io.ObjectStreamClass.getSerialFields(ObjectStreamClass.java:1773) at java.io.ObjectStreamClass.access$800(ObjectStreamClass.java:79) at java.io.ObjectStreamClass$3.run(ObjectStreamClass.java:508) at java.io.ObjectStreamClass$3.run(ObjectStreamClass.java:494) at java.security.AccessController.doPrivileged(Native Method) at java.io.ObjectStreamClass.<init>(ObjectStreamClass.java:494) at java.io.ObjectStreamClass.lookup(ObjectStreamClass.java:391) at java.io.ObjectStreamClass.initNonProxy(ObjectStreamClass.java:681) at java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1942) at java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1808) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2099) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1625) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2344) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2268) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2126) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1625) at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2344) at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2268) at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2126) at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1625) at java.io.ObjectInputStream.readObject(ObjectInputStream.java:465) at java.io.ObjectInputStream.readObject(ObjectInputStream.java:423) at hudson.remoting.UserRequest.deserialize(UserRequest.java:290) at hudson.remoting.UserRequest.perform(UserRequest.java:189) at hudson.remoting.UserRequest.perform(UserRequest.java:54) at hudson.remoting.Request$2.run(Request.java:369) at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at hudson.remoting.Engine$1.lambda$newThread$0(Engine.java:117) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.ClassNotFoundException: hudson.util.io.Archiver at java.net.URLClassLoader.findClass(URLClassLoader.java:382) at hudson.remoting.RemoteClassLoader.findClass(RemoteClassLoader.java:173) at java.lang.ClassLoader.loadClass(ClassLoader.java:418) at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ... 36 more
Both the master and agent are running on JDK8
Agent (A VM)
openjdk version "1.8.0_252" OpenJDK Runtime Environment (build 1.8.0_252-b09) OpenJDK 64-Bit Server VM (build 25.252-b09, mixed mode)
Master (Official docker container)
openjdk version "1.8.0_242" OpenJDK Runtime Environment (build 1.8.0_242-b08) OpenJDK 64-Bit Server VM (build 25.242-b08, mixed mode)
If you have any idea about what is causing the issue.
PS : I didn't had time to test using latest jenkins version and remoting 4.3. I don't know if it will change anything
PS2: Totally aware that Websocket are still in beta
Thanks!
- relates to
-
JENKINS-61409 Websocket connections crash if message size is greater than 64Kb
- Resolved
-
JENKINS-64598 Jenkins agent disconnects on k8s with SIGHUP / ClosedChannelException
- Closed