Major When branch indexing is run either manually or by timer we get a deadlock. This slows down the Jenkins and eventually makes it crash(until we restart).
This is the stack traces from the thread dump. Would this require a change in the lock orders or locking the runMap together with the multibranch project?
Found one Java-level deadlock:
=============================
"jenkins.util.Timer 1":
waiting to lock monitor 0x00007f76b6453380 (object 0x00000000e61629c0, a org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject),
which is held by "Executor #-1 for Built-In Node : executing BranchIndexing[MY-JENKINS-JOB]"
"Executor #-1 for Built-In Node : executing BranchIndexing[MY-JENKINS-JOB]":
waiting to lock monitor 0x00007f7e380c9c00 (object 0x00000000c569da30, a hudson.model.RunMap),
which is held by "jenkins.util.Timer 1"
Java stack information for the threads listed above:
===================================================
"jenkins.util.Timer 1":
at jenkins.branch.MultiBranchProject.getProjectFactory(MultiBranchProject.java:336)
- waiting to lock <0x00000000e61629c0> (a org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject)
at io.jenkins.blueocean.rest.impl.pipeline.MultiBranchPipelineImpl$PipelineFactoryImpl.getPipeline(MultiBranchPipelineImpl.java:276)
at io.jenkins.blueocean.rest.impl.pipeline.MultiBranchPipelineImpl$PipelineFactoryImpl.resolve(MultiBranchPipelineImpl.java:291)
at io.jenkins.blueocean.service.embedded.rest.PipelineFolderImpl$PipelineFactoryImpl.resolve(PipelineFolderImpl.java:174)
at io.jenkins.blueocean.service.embedded.rest.PipelineFolderImpl$PipelineFactoryImpl.resolve(PipelineFolderImpl.java:174)
at io.jenkins.blueocean.service.embedded.rest.PipelineFolderImpl$PipelineFactoryImpl.resolve(PipelineFolderImpl.java:174)
at io.jenkins.blueocean.rest.factory.BluePipelineFactory.resolve(BluePipelineFactory.java:75)
at io.jenkins.blueocean.service.embedded.BlueOceanUrlMapperImpl.getJobResource(BlueOceanUrlMapperImpl.java:80)
at io.jenkins.blueocean.service.embedded.BlueOceanUrlMapperImpl.getUrl(BlueOceanUrlMapperImpl.java:49)
at io.jenkins.blueocean.service.embedded.BlueOceanUrlObjectImpl.computeUrl(BlueOceanUrlObjectImpl.java:42)
at io.jenkins.blueocean.service.embedded.BlueOceanUrlObjectImpl.<init>(BlueOceanUrlObjectImpl.java:21)
at io.jenkins.blueocean.service.embedded.BlueOceanUrlObjectFactoryImpl.get(BlueOceanUrlObjectFactoryImpl.java:18)
at io.jenkins.blueocean.service.embedded.TryBlueOceanMenu.createFor(TryBlueOceanMenu.java:50)
at io.jenkins.blueocean.service.embedded.TryBlueOceanMenu.createFor(TryBlueOceanMenu.java:26)
at hudson.model.Actionable.createFor(Actionable.java:115)
at hudson.model.Actionable.getAllActions(Actionable.java:101)
at hudson.model.Run.onLoad(Run.java:386)
at org.jenkinsci.plugins.workflow.job.WorkflowRun.onLoad(WorkflowRun.java:566) - locked <0x000000050cfdc4a8> (a java.lang.Object)
at hudson.model.RunMap.retrieve(RunMap.java:233)
at hudson.model.RunMap.retrieve(RunMap.java:61)
at jenkins.model.lazy.AbstractLazyLoadRunMap.load(AbstractLazyLoadRunMap.java:650)
at jenkins.model.lazy.AbstractLazyLoadRunMap.load(AbstractLazyLoadRunMap.java:632)
at jenkins.model.lazy.AbstractLazyLoadRunMap.getByNumber(AbstractLazyLoadRunMap.java:530) - locked <0x00000000c569da30> (a hudson.model.RunMap)
at jenkins.model.lazy.AbstractLazyLoadRunMap.search(AbstractLazyLoadRunMap.java:495)
at jenkins.model.lazy.AbstractLazyLoadRunMap.newestBuild(AbstractLazyLoadRunMap.java:428)
at jenkins.model.lazy.LazyBuildMixIn.getLastBuild(LazyBuildMixIn.java:251)
at org.jenkinsci.plugins.workflow.job.WorkflowJob.getLastBuild(WorkflowJob.java:242)
at org.jenkinsci.plugins.workflow.job.WorkflowJob.getLastBuild(WorkflowJob.java:105)
at hudson.model.Job.isLogUpdated(Job.java:314)
at org.jenkinsci.plugins.workflow.job.WorkflowJob.getCauseOfBlockage(WorkflowJob.java:306)
at hudson.model.Queue.getCauseOfBlockageForTask(Queue.java:1239)
at hudson.model.Queue.getCauseOfBlockageForItem(Queue.java:1196)
at hudson.model.Queue.maintain(Queue.java:1570)
at hudson.model.Queue$MaintainTask.doRun(Queue.java:2912)
at hudson.triggers.SafeTimerTask.run(SafeTimerTask.java:92)
at jenkins.security.ImpersonatingScheduledExecutorService$1.run(ImpersonatingScheduledExecutorService.java:67)
at java.util.concurrent.Executors$RunnableAdapter.call(java.base@11.0.19/Unknown Source)
at java.util.concurrent.FutureTask.runAndReset(java.base@11.0.19/Unknown Source)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(java.base@11.0.19/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@11.0.19/Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@11.0.19/Unknown Source)
at java.lang.Thread.run(java.base@11.0.19/Unknown Source)
"Executor #-1 for Built-In Node : executing BranchIndexing[MY-JENKINS-JOB]":
at jenkins.model.lazy.AbstractLazyLoadRunMap.getByNumber(AbstractLazyLoadRunMap.java:519) - waiting to lock <0x00000000c569da30> (a hudson.model.RunMap)
at jenkins.model.lazy.AbstractLazyLoadRunMap.search(AbstractLazyLoadRunMap.java:495)
at jenkins.model.lazy.AbstractLazyLoadRunMap.newestBuild(AbstractLazyLoadRunMap.java:428)
at jenkins.model.lazy.LazyBuildMixIn.getLastBuild(LazyBuildMixIn.java:251)
at org.jenkinsci.plugins.workflow.job.WorkflowJob.getLastBuild(WorkflowJob.java:242)
at org.jenkinsci.plugins.workflow.job.WorkflowJob.getLastBuild(WorkflowJob.java:105)
at hudson.model.Job.getBuildHealthReports(Job.java:1203)
at hudson.model.Job.getBuildHealth(Job.java:1196)
at com.cloudbees.hudson.plugins.folder.health.FolderHealthMetric.getHealthReport(FolderHealthMetric.java:48)
at com.cloudbees.hudson.plugins.folder.health.WorstChildHealthMetric$ReporterImpl.observe(WorstChildHealthMetric.java:86)
at com.cloudbees.hudson.plugins.folder.AbstractFolder.getBuildHealthReports(AbstractFolder.java:924)
at java.lang.invoke.DirectMethodHandle$Holder.invokeVirtual(java.base@11.0.19/DirectMethodHandle$Holder)
at java.lang.invoke.LambdaForm$MH/0x00000008011c9c40.invoke(java.base@11.0.19/LambdaForm$MH)
at java.lang.invoke.LambdaForm$MH/0x000000080007fc40.invoke_MT(java.base@11.0.19/LambdaForm$MH)
at org.kohsuke.stapler.export.MethodProperty.getValue(MethodProperty.java:76)
at org.kohsuke.stapler.export.ExportInterceptor$1.getValue(ExportInterceptor.java:46)
at io.jenkins.plugins.generic.event.json.ExportedBeanProcessor$IgnoreURLExportInterceptor.getValue(ExportedBeanProcessor.java:71)
at org.kohsuke.stapler.export.Property.writeTo(Property.java:136)
at org.kohsuke.stapler.export.Model.writeNestedObjectTo(Model.java:222)
at org.kohsuke.stapler.export.Model.writeNestedObjectTo(Model.java:218)
at org.kohsuke.stapler.export.Model.writeNestedObjectTo(Model.java:218)
at org.kohsuke.stapler.export.Model.writeNestedObjectTo(Model.java:218)
at org.kohsuke.stapler.export.Model.writeTo(Model.java:193)
at org.kohsuke.stapler.export.Model.writeTo(Model.java:213)
at org.kohsuke.stapler.export.Model.writeTo(Model.java:181)
at io.jenkins.plugins.generic.event.json.ExportedBeanProcessor.processBean(ExportedBeanProcessor.java:43)
at net.sf.json.JSONObject._fromBean(JSONObject.java:619)
at net.sf.json.JSONObject.fromObject(JSONObject.java:169)
at net.sf.json.AbstractJSON._processValue(AbstractJSON.java:250)
at net.sf.json.JSONObject._processValue(JSONObject.java:2599)
at net.sf.json.JSONObject.processValue(JSONObject.java:2665)
at net.sf.json.JSONObject.setInternal(JSONObject.java:2680)
at net.sf.json.JSONObject.setValue(JSONObject.java:1376)
at net.sf.json.JSONObject.defaultBeanProcessing(JSONObject.java:708)
at net.sf.json.JSONObject._fromBean(JSONObject.java:642)
at net.sf.json.JSONObject.fromObject(JSONObject.java:169)
at io.jenkins.plugins.generic.event.HttpEventSender$CustomCloseableHttpAsyncClient.sendPost(HttpEventSender.java:90)
at io.jenkins.plugins.generic.event.HttpEventSender.send(HttpEventSender.java:45)
at io.jenkins.plugins.generic.event.listener.GenericEventItemListener.onUpdated(GenericEventItemListener.java:118)
at hudson.model.listeners.ItemListener.lambda$fireOnUpdated$2(ItemListener.java:205)
at hudson.model.listeners.ItemListener$$Lambda$2988/0x000000080359d440.accept(Unknown Source)
at jenkins.util.Listeners.lambda$notify$0(Listeners.java:59)
at jenkins.util.Listeners$$Lambda$777/0x0000000801619840.run(Unknown Source)
at jenkins.util.Listeners.notify(Listeners.java:70)
at hudson.model.listeners.ItemListener.fireOnUpdated(ItemListener.java:205)
at com.cloudbees.hudson.plugins.folder.AbstractFolder.save(AbstractFolder.java:1317) - locked <0x00000000e61629c0> (a org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject)
at com.cloudbees.hudson.plugins.folder.AbstractFolder.onDeleted(AbstractFolder.java:1085)
at com.cloudbees.hudson.plugins.folder.AbstractFolder.onDeleted(AbstractFolder.java:145)
at hudson.model.AbstractItem.delete(AbstractItem.java:805)
at hudson.model.Job.delete(Job.java:684)
at com.cloudbees.hudson.plugins.folder.computed.ComputedFolder.updateChildren(ComputedFolder.java:285)
at com.cloudbees.hudson.plugins.folder.computed.FolderComputation.run(FolderComputation.java:167)
at jenkins.branch.MultiBranchProject$BranchIndexing.run(MultiBranchProject.java:1032)
at hudson.model.ResourceController.execute(ResourceController.java:101)
at hudson.model.Executor.run(Executor.java:442)
Found 1 deadlock.
