-
Bug
-
Resolution: Unresolved
-
Minor
-
None
-
Jenkins: 2.287
OS: Linux - 5.4.0-1043-azure
---
jenkins-design-language:1.24.5
javadoc:1.6
sse-gateway:1.24
workflow-multibranch:2.23
kubernetes:1.29.2
echarts-api:5.0.2-1
configuration-as-code:1.47
metrics:4.0.2.7
lockable-resources:2.10
git-client:3.7.1
github-branch-source:2.10.3-rc1417.95f9d4c176bb
authentication-tokens:1.4
slack:2.46
workflow-durable-task-step:2.38
favorite:2.3.3
kubernetes-credentials:0.8.0
ansicolor:0.7.5
blueocean:1.24.5
momentjs:1.1.1
matrix-project:1.18
blueocean-i18n:1.24.5
cloudbees-folder:6.15
popper-api:1.16.1-2
jackson2-api:2.12.1
cloudbees-disk-usage-simple:0.10
workflow-basic-steps:2.23
dark-theme:0.0.12
azure-keyvault:116.v4c0cbc857be0
jjwt-api:0.11.2-9.c8b45b8bb173
sauce-ondemand:1.194
script-security:1.76
blueocean-pipeline-scm-api:1.24.5
pubsub-light:1.13
pipeline-build-step:2.13
git:4.7.1
blueocean-bitbucket-pipeline:1.24.5
github-api:1.123
token-macro:2.15
workflow-step-api:2.23
okhttp-api:3.14.9
kubernetes-client-api:4.13.2-1
command-launcher:1.5
prometheus:2.0.9
job-dsl:1.77
plain-credentials:1.7
branch-api:2.6.3
blueocean-display-url:2.4.1
antisamy-markup-formatter:2.1
pipeline-model-extensions:1.8.4
blueocean-git-pipeline:1.24.5
bouncycastle-api:2.20
pipeline-stage-view:2.19
pipeline-milestone-step:1.3.2
blueocean-pipeline-api-impl:1.24.5
pipeline-rest-api:2.19
github-checks:1.0.12
junit:1.49
workflow-cps:2.90
github-scm-trait-notification-context:1.1
maven-plugin:3.10
monitoring:1.87.0
jdk-tool:1.5
blueocean-jwt:1.24.5
azure-commons:1.1.2
gatling:1.3.0
workflow-support:3.8
handlebars:3.0.8
sonar:2.14-PR-182-1
pipeline-stage-step:2.5
pipeline-model-api:1.8.4
handy-uri-templates-2-api:2.1.8-1.0
pipeline-utility-steps:2.7.1
bootstrap4-api:4.6.0-3
http_request:1.8.27
workflow-cps-global-lib:2.18
cloudbees-bitbucket-branch-source:2.9.7
apache-httpcomponents-client-4-api:4.5.13-1.0
plugin-util-api:2.1.0
docker-commons:1.17
blueocean-web:1.24.5
blueocean-events:1.24.5
pipeline-model-definition:1.8.4
git-server:1.9
variant:1.4
structs:1.22
workflow-job:2.40
azure-credentials:177.v816b81058012
blueocean-github-pipeline:1.24.5
credentials-binding:1.24
blueocean-core-js:1.24.5
pipeline-graph-analysis:1.10
timestamper:1.12
htmlpublisher:1.25
checks-api:1.7.0
theme-manager:0.6
pipeline-github-lib:1.0
ssh-credentials:1.18.2
sshd:3.0.3
blueocean-commons:1.24.5
jquery3-api:3.6.0-1
build-monitor-plugin:1.12+build.201809061734
blueocean-personalization:1.24.5
trilead-api:1.0.13
jacoco:3.1.1
font-awesome-api:5.15.2-2
workflow-aggregator:2.6
blueocean-autofavorite:1.2.4
mailer:1.34
snakeyaml-api:1.27.0
workflow-api:2.42
basic-branch-build-strategies:1.3.2
github:1.33.1
cloud-stats:0.27
pipeline-input-step:2.12
docker-workflow:1.26
scm-api:2.6.4
durable-task:1.35
blueocean-dashboard:1.24.5
saml:2.0.2
matrix-auth:2.6.6
azure-vm-agents:768.vb8b0d31eef4f
jsch:0.1.55.2
display-url-api:2.3.4
credentials:2.3.15
extended-read-permission:3.2
blueocean-rest-impl:1.24.5
jaxb:2.3.0.1
blueocean-config:1.24.5
blueocean-rest:1.24.5
blueocean-pipeline-editor:1.24.5
run-condition:1.5
pipeline-stage-tags-metadata:1.8.4
ace-editor:1.1
workflow-scm-step:2.12Jenkins: 2.287 OS: Linux - 5.4.0-1043-azure --- jenkins-design-language:1.24.5 javadoc:1.6 sse-gateway:1.24 workflow-multibranch:2.23 kubernetes:1.29.2 echarts-api:5.0.2-1 configuration-as-code:1.47 metrics:4.0.2.7 lockable-resources:2.10 git-client:3.7.1 github-branch-source:2.10.3-rc1417.95f9d4c176bb authentication-tokens:1.4 slack:2.46 workflow-durable-task-step:2.38 favorite:2.3.3 kubernetes-credentials:0.8.0 ansicolor:0.7.5 blueocean:1.24.5 momentjs:1.1.1 matrix-project:1.18 blueocean-i18n:1.24.5 cloudbees-folder:6.15 popper-api:1.16.1-2 jackson2-api:2.12.1 cloudbees-disk-usage-simple:0.10 workflow-basic-steps:2.23 dark-theme:0.0.12 azure-keyvault:116.v4c0cbc857be0 jjwt-api:0.11.2-9.c8b45b8bb173 sauce-ondemand:1.194 script-security:1.76 blueocean-pipeline-scm-api:1.24.5 pubsub-light:1.13 pipeline-build-step:2.13 git:4.7.1 blueocean-bitbucket-pipeline:1.24.5 github-api:1.123 token-macro:2.15 workflow-step-api:2.23 okhttp-api:3.14.9 kubernetes-client-api:4.13.2-1 command-launcher:1.5 prometheus:2.0.9 job-dsl:1.77 plain-credentials:1.7 branch-api:2.6.3 blueocean-display-url:2.4.1 antisamy-markup-formatter:2.1 pipeline-model-extensions:1.8.4 blueocean-git-pipeline:1.24.5 bouncycastle-api:2.20 pipeline-stage-view:2.19 pipeline-milestone-step:1.3.2 blueocean-pipeline-api-impl:1.24.5 pipeline-rest-api:2.19 github-checks:1.0.12 junit:1.49 workflow-cps:2.90 github-scm-trait-notification-context:1.1 maven-plugin:3.10 monitoring:1.87.0 jdk-tool:1.5 blueocean-jwt:1.24.5 azure-commons:1.1.2 gatling:1.3.0 workflow-support:3.8 handlebars:3.0.8 sonar:2.14-PR-182-1 pipeline-stage-step:2.5 pipeline-model-api:1.8.4 handy-uri-templates-2-api:2.1.8-1.0 pipeline-utility-steps:2.7.1 bootstrap4-api:4.6.0-3 http_request:1.8.27 workflow-cps-global-lib:2.18 cloudbees-bitbucket-branch-source:2.9.7 apache-httpcomponents-client-4-api:4.5.13-1.0 plugin-util-api:2.1.0 docker-commons:1.17 blueocean-web:1.24.5 blueocean-events:1.24.5 pipeline-model-definition:1.8.4 git-server:1.9 variant:1.4 structs:1.22 workflow-job:2.40 azure-credentials:177.v816b81058012 blueocean-github-pipeline:1.24.5 credentials-binding:1.24 blueocean-core-js:1.24.5 pipeline-graph-analysis:1.10 timestamper:1.12 htmlpublisher:1.25 checks-api:1.7.0 theme-manager:0.6 pipeline-github-lib:1.0 ssh-credentials:1.18.2 sshd:3.0.3 blueocean-commons:1.24.5 jquery3-api:3.6.0-1 build-monitor-plugin:1.12+build.201809061734 blueocean-personalization:1.24.5 trilead-api:1.0.13 jacoco:3.1.1 font-awesome-api:5.15.2-2 workflow-aggregator:2.6 blueocean-autofavorite:1.2.4 mailer:1.34 snakeyaml-api:1.27.0 workflow-api:2.42 basic-branch-build-strategies:1.3.2 github:1.33.1 cloud-stats:0.27 pipeline-input-step:2.12 docker-workflow:1.26 scm-api:2.6.4 durable-task:1.35 blueocean-dashboard:1.24.5 saml:2.0.2 matrix-auth:2.6.6 azure-vm-agents:768.vb8b0d31eef4f jsch:0.1.55.2 display-url-api:2.3.4 credentials:2.3.15 extended-read-permission:3.2 blueocean-rest-impl:1.24.5 jaxb:2.3.0.1 blueocean-config:1.24.5 blueocean-rest:1.24.5 blueocean-pipeline-editor:1.24.5 run-condition:1.5 pipeline-stage-tags-metadata:1.8.4 ace-editor:1.1 workflow-scm-step:2.12
We're seeing huge delays in processing events from GitHub (at least 2 hours from PR open to build scheduled).
PushGHEventSubscriber is where this begins.
SCMEvent defines a thread pool with a max size of 10.
I've attached a set of thread dumps, but you can see in this excerpt that all 10 threads are sleeping waiting for rate limiting
This does not seem correct as we are running with GitHubApps and approximately 15000 api calls are allowed.
I've checked a few of our GitHub apps and all of them have over 12,000 calls remaining.
cc bitwiseman carroll dnusbaum
I believe this behaviour only started after we upgraded to https://github.com/jenkinsci/github-branch-source-plugin/releases/tag/github-branch-source-2.10.1.
I'm going to try either downgrade or revert the specific changes tomorrow to get us past this.
class org.jenkinsci.plugins.github_branch_source.PullRequestGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:50:33 UTC 2021 / SCMEvent [#7]sleeping , holding [ 0x00000004b8a01310 ] class org.jenkinsci.plugins.github_branch_source.PullRequestGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:51:10 UTC 2021 / SCMEvent [#6]sleeping , holding [ 0x00000004aa801628 ] class org.jenkinsci.plugins.github_branch_source.PullRequestGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:51:33 UTC 2021 / SCMEvent [#1]sleeping , holding [ 0x0000000489521128 ] class org.jenkinsci.plugins.github_branch_source.PullRequestGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:51:48 UTC 2021 / SCMEvent [#9]sleeping , holding [ 0x00000004aa2001d0 ] class org.jenkinsci.plugins.github_branch_source.PullRequestGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:52:38 UTC 2021 / SCMEvent [#5]sleeping , holding [ 0x00000004a79cbb88 ] class org.jenkinsci.plugins.github_branch_source.PushGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:50:49 UTC 2021 / SCMEvent [#3]sleeping , holding [ 0x0000000488800178 ] class org.jenkinsci.plugins.github_branch_source.PushGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:51:21 UTC 2021 / SCMEvent [#2]sleeping , holding [ 0x0000000494c00220 ] class org.jenkinsci.plugins.github_branch_source.PushGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:51:32 UTC 2021 / SCMEvent [#8]sleeping , holding [ 0x00000004b8a02290 ] class org.jenkinsci.plugins.github_branch_source.PushGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:52:15 UTC 2021 / SCMEvent [#10]sleeping , holding [ 0x0000000488a02170 ] class org.jenkinsci.plugins.github_branch_source.PushGHEventSubscriber$SCMHeadEventImpl Thu Apr 08 12:52:37 UTC 2021 / SCMEvent [#4]sleeping , holding [ 0x00000004a7961600 ] at java.lang.Thread.sleep(java.base@11.0.10/Native Method) at org.jenkinsci.plugins.github_branch_source.ApiRateLimitChecker$LocalChecker.waitUntilRateLimit(ApiRateLimitChecker.java:323) at org.jenkinsci.plugins.github_branch_source.ApiRateLimitChecker$LocalChecker.checkRateLimit(ApiRateLimitChecker.java:259) at org.jenkinsci.plugins.github_branch_source.ApiRateLimitChecker$RateLimitCheckerAdapter.checkRateLimit(ApiRateLimitChecker.java:240) at org.kohsuke.github.GitHubRateLimitChecker.checkRateLimit(GitHubRateLimitChecker.java:126) at org.kohsuke.github.GitHubClient.sendRequest(GitHubClient.java:392) at org.kohsuke.github.GitHubClient.sendRequest(GitHubClient.java:358) at org.kohsuke.github.Requester.fetch(Requester.java:76) at org.kohsuke.github.GHRepository.read(GHRepository.java:119) at org.kohsuke.github.GHPerson.getRepository(GHPerson.java:156) at org.jenkinsci.plugins.github_branch_source.GitHubSCMNavigator.visitSource(GitHubSCMNavigator.java:1313) at org.jenkinsci.plugins.github_branch_source.GitHubSCMNavigator.visitSources(GitHubSCMNavigator.java:915) at jenkins.scm.api.SCMNavigator.visitSources(SCMNavigator.java:221) at jenkins.branch.OrganizationFolder$SCMEventListenerImpl.onSCMHeadEvent(OrganizationFolder.java:1165) at jenkins.scm.api.SCMHeadEvent$DispatcherImpl.fire(SCMHeadEvent.java:246) at jenkins.scm.api.SCMHeadEvent$DispatcherImpl.fire(SCMHeadEvent.java:229) at jenkins.scm.api.SCMEvent$Dispatcher.run(SCMEvent.java:505) at jenkins.security.ImpersonatingScheduledExecutorService$1.run(ImpersonatingScheduledExecutorService.java:67) at java.util.concurrent.Executors$RunnableAdapter.call(java.base@11.0.10/Executors.java:515) at java.lang.Thread.run(java.base@11.0.10/Thread.java:834)