-
Bug
-
Resolution: Fixed
-
Major
-
None
-
Jenkins: 2.283
OS: Linux - (docker)
---
jenkins-design-language:1.24.4
javadoc:1.6
sse-gateway:1.24
workflow-multibranch:2.22
kubernetes:1.29.2
echarts-api:5.0.1-1
configuration-as-code:1.47
metrics:4.0.2.7
lockable-resources:2.10
git-client:3.6.0
github-branch-source:2.10.2
authentication-tokens:1.4
slack:2.45
workflow-durable-task-step:2.38
favorite:2.3.3
kubernetes-credentials:0.8.0
ansicolor:0.7.5
blueocean:1.24.4
momentjs:1.1.1
matrix-project:1.18
blueocean-i18n:1.24.4
cloudbees-folder:6.15
popper-api:1.16.1-2
jackson2-api:2.12.1
cloudbees-disk-usage-simple:0.10
workflow-basic-steps:2.23
dark-theme:0.0.12
azure-keyvault:2.2
jjwt-api:0.11.2-9.c8b45b8bb173
sauce-ondemand:1.194
script-security:1.76
blueocean-pipeline-scm-api:1.24.4
pubsub-light:1.13
pipeline-build-step:2.13
git:4.6.0
blueocean-bitbucket-pipeline:1.24.4
github-api:1.123
token-macro:2.15
workflow-step-api:2.23
okhttp-api:3.14.9
kubernetes-client-api:4.13.2-1
command-launcher:1.5
prometheus:2.0.8
job-dsl:1.77
plain-credentials:1.7
branch-api:2.6.3
blueocean-display-url:2.4.1
antisamy-markup-formatter:2.1
pipeline-model-extensions:1.8.4
blueocean-git-pipeline:1.24.4
bouncycastle-api:2.20
pipeline-stage-view:2.19
pipeline-milestone-step:1.3.2
blueocean-pipeline-api-impl:1.24.4
pipeline-rest-api:2.19
github-checks:1.0.10
junit:1.48
workflow-cps:2.90
github-scm-trait-notification-context:1.1
maven-plugin:3.10
monitoring:1.86.0
jdk-tool:1.5
blueocean-jwt:1.24.4
azure-commons:1.1.1
gatling:1.3.0
workflow-support:3.8
handlebars:1.1.1
sonar:2.11-hmcts
pipeline-stage-step:2.5
pipeline-model-api:1.8.4
handy-uri-templates-2-api:2.1.8-1.0
pipeline-utility-steps:2.6.1
bootstrap4-api:4.6.0-2
http_request:1.8.27
workflow-cps-global-lib:2.18
cloudbees-bitbucket-branch-source:2.9.7
apache-httpcomponents-client-4-api:4.5.13-1.0
plugin-util-api:2.0.0
docker-commons:1.17
blueocean-web:1.24.4
blueocean-events:1.24.4
pipeline-model-definition:1.8.4
git-server:1.9
variant:1.4
structs:1.22
workflow-job:2.40
azure-credentials:4.0.6
blueocean-github-pipeline:1.24.4
credentials-binding:1.24
blueocean-core-js:1.24.4
pipeline-graph-analysis:1.10
timestamper:1.11.8
htmlpublisher:1.25
checks-api:1.5.0
theme-manager:0.6
pipeline-github-lib:1.0
ssh-credentials:1.18.1
sshd:3.0.3
blueocean-commons:1.24.4
jquery3-api:3.5.1-3
build-monitor-plugin:1.12+build.201809061734
blueocean-personalization:1.24.4
trilead-api:1.0.13
jacoco:3.1.1
font-awesome-api:5.15.2-2
workflow-aggregator:2.6
blueocean-autofavorite:1.2.4
mailer:1.33
snakeyaml-api:1.27.0
workflow-api:2.41
basic-branch-build-strategies:1.3.2
github:1.33.1
cloud-stats:0.26
pipeline-input-step:2.12
docker-workflow:1.26
scm-api:2.6.4
durable-task:1.35
blueocean-dashboard:1.24.4
saml:2.0.0
matrix-auth:2.6.5
azure-vm-agents:1.5.3
jsch:0.1.55.2
display-url-api:2.3.4
credentials:2.3.15
extended-read-permission:3.2
blueocean-rest-impl:1.24.4
jaxb:2.3.0.1
blueocean-config:1.24.4
blueocean-rest:1.24.4
blueocean-pipeline-editor:1.24.4
run-condition:1.5
pipeline-stage-tags-metadata:1.8.4
ace-editor:1.1
workflow-scm-step:2.12Jenkins: 2.283 OS: Linux - (docker) --- jenkins-design-language:1.24.4 javadoc:1.6 sse-gateway:1.24 workflow-multibranch:2.22 kubernetes:1.29.2 echarts-api:5.0.1-1 configuration-as-code:1.47 metrics:4.0.2.7 lockable-resources:2.10 git-client:3.6.0 github-branch-source:2.10.2 authentication-tokens:1.4 slack:2.45 workflow-durable-task-step:2.38 favorite:2.3.3 kubernetes-credentials:0.8.0 ansicolor:0.7.5 blueocean:1.24.4 momentjs:1.1.1 matrix-project:1.18 blueocean-i18n:1.24.4 cloudbees-folder:6.15 popper-api:1.16.1-2 jackson2-api:2.12.1 cloudbees-disk-usage-simple:0.10 workflow-basic-steps:2.23 dark-theme:0.0.12 azure-keyvault:2.2 jjwt-api:0.11.2-9.c8b45b8bb173 sauce-ondemand:1.194 script-security:1.76 blueocean-pipeline-scm-api:1.24.4 pubsub-light:1.13 pipeline-build-step:2.13 git:4.6.0 blueocean-bitbucket-pipeline:1.24.4 github-api:1.123 token-macro:2.15 workflow-step-api:2.23 okhttp-api:3.14.9 kubernetes-client-api:4.13.2-1 command-launcher:1.5 prometheus:2.0.8 job-dsl:1.77 plain-credentials:1.7 branch-api:2.6.3 blueocean-display-url:2.4.1 antisamy-markup-formatter:2.1 pipeline-model-extensions:1.8.4 blueocean-git-pipeline:1.24.4 bouncycastle-api:2.20 pipeline-stage-view:2.19 pipeline-milestone-step:1.3.2 blueocean-pipeline-api-impl:1.24.4 pipeline-rest-api:2.19 github-checks:1.0.10 junit:1.48 workflow-cps:2.90 github-scm-trait-notification-context:1.1 maven-plugin:3.10 monitoring:1.86.0 jdk-tool:1.5 blueocean-jwt:1.24.4 azure-commons:1.1.1 gatling:1.3.0 workflow-support:3.8 handlebars:1.1.1 sonar:2.11-hmcts pipeline-stage-step:2.5 pipeline-model-api:1.8.4 handy-uri-templates-2-api:2.1.8-1.0 pipeline-utility-steps:2.6.1 bootstrap4-api:4.6.0-2 http_request:1.8.27 workflow-cps-global-lib:2.18 cloudbees-bitbucket-branch-source:2.9.7 apache-httpcomponents-client-4-api:4.5.13-1.0 plugin-util-api:2.0.0 docker-commons:1.17 blueocean-web:1.24.4 blueocean-events:1.24.4 pipeline-model-definition:1.8.4 git-server:1.9 variant:1.4 structs:1.22 workflow-job:2.40 azure-credentials:4.0.6 blueocean-github-pipeline:1.24.4 credentials-binding:1.24 blueocean-core-js:1.24.4 pipeline-graph-analysis:1.10 timestamper:1.11.8 htmlpublisher:1.25 checks-api:1.5.0 theme-manager:0.6 pipeline-github-lib:1.0 ssh-credentials:1.18.1 sshd:3.0.3 blueocean-commons:1.24.4 jquery3-api:3.5.1-3 build-monitor-plugin:1.12+build.201809061734 blueocean-personalization:1.24.4 trilead-api:1.0.13 jacoco:3.1.1 font-awesome-api:5.15.2-2 workflow-aggregator:2.6 blueocean-autofavorite:1.2.4 mailer:1.33 snakeyaml-api:1.27.0 workflow-api:2.41 basic-branch-build-strategies:1.3.2 github:1.33.1 cloud-stats:0.26 pipeline-input-step:2.12 docker-workflow:1.26 scm-api:2.6.4 durable-task:1.35 blueocean-dashboard:1.24.4 saml:2.0.0 matrix-auth:2.6.5 azure-vm-agents:1.5.3 jsch:0.1.55.2 display-url-api:2.3.4 credentials:2.3.15 extended-read-permission:3.2 blueocean-rest-impl:1.24.4 jaxb:2.3.0.1 blueocean-config:1.24.4 blueocean-rest:1.24.4 blueocean-pipeline-editor:1.24.4 run-condition:1.5 pipeline-stage-tags-metadata:1.8.4 ace-editor:1.1 workflow-scm-step:2.12
-
-
github-branch-source-2.10.3
We upgraded to 2.283 of Jenkins from ~2.279 3 weeks ago.
Since the last 1-2 weeks we've noticed builds often not being triggered when they should be, if you click 'Build now' then it flashes for a second but no job starts.
The job does start eventually I believe but minutes later.
Attached results from 'collectPerformanceData.sh' in https://support.cloudbees.com/hc/en-us/articles/229795948-Required-Data-CJP-CJT-Hang-Issue-On-Linux
Note most of the tools the script wants isn't installed as its running in a docker container.
I ran thread dumps every 5 seconds for 3 minutes
you can see mostly of the relevant info with
tar -xzf performanceData.7.output.tar.gz cd threads grep -A 10 -B 10 hudson.model.Queue.withLock *
I'm not sure exactly what I'm looking for but here's an excerpt
threads.7.20210330084345.txt-"Executor #-1 for master" #334602 daemon prio=5 os_prio=0 cpu=0.09ms elapsed=13.88s tid=0x00007f9e04604000 nid=0x5713 waiting on condition [0x00007f9d4717f000] threads.7.20210330084345.txt- java.lang.Thread.State: WAITING (parking) threads.7.20210330084345.txt- at jdk.internal.misc.Unsafe.park(java.base@11.0.10/Native Method) threads.7.20210330084345.txt- - parking to wait for <0x000000046cdf2d10> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) threads.7.20210330084345.txt- at java.util.concurrent.locks.LockSupport.park(java.base@11.0.10/LockSupport.java:194) threads.7.20210330084345.txt- at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(java.base@11.0.10/AbstractQueuedSynchronizer.java:885) threads.7.20210330084345.txt- at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(java.base@11.0.10/AbstractQueuedSynchronizer.java:917) threads.7.20210330084345.txt- at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@11.0.10/AbstractQueuedSynchronizer.java:1240) threads.7.20210330084345.txt- at java.util.concurrent.locks.ReentrantLock.lock(java.base@11.0.10/ReentrantLock.java:267) threads.7.20210330084345.txt- at hudson.model.Queue._withLock(Queue.java:1444) threads.7.20210330084345.txt: at hudson.model.Queue.withLock(Queue.java:1304) threads.7.20210330084345.txt- at hudson.model.Executor.run(Executor.java:347) threads.7.20210330084345.txt- threads.7.20210330084345.txt- Locked ownable synchronizers: threads.7.20210330084345.txt- - None threads.7.20210330084345.txt- threads.7.20210330084345.txt-"org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep [#19471]" #334603 daemon prio=5 os_prio=0 cpu=0.24ms elapsed=13.45s tid=0x00007f9ebc658000 nid=0x5714 waiting on condition [0x00007f9ded3da000] threads.7.20210330084345.txt- java.lang.Thread.State: TIMED_WAITING (parking) threads.7.20210330084345.txt- at jdk.internal.misc.Unsafe.park(java.base@11.0.10/Native Method) threads.7.20210330084345.txt- - parking to wait for <0x0000000477c00dc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) threads.7.20210330084345.txt- at java.util.concurrent.locks.LockSupport.parkNanos(java.base@11.0.10/LockSupport.java:234)
