-
Bug
-
Resolution: Unresolved
-
Major
-
None
Steps to Reproduce
- Install a brand new controller using our Jenkins job.
- Look for the plugin updates, install them manually, and click on the restart option at the bottom.
- If there are issues with plugins, then we will get into this situation of the Jenkins application being unreachable or down.
ound one Java-level deadlock: ============================= "Handling GET /pbtest/ from 10.0.0.2 : Jetty (winstone)-14": waiting to lock monitor 0x00007f4e4c824870 (object 0x000000070874b4f0, a hudson.ExtensionList$Lock), which is held by "GitLabConnection.migrate""GitLabConnection.migrate": waiting for ownable synchronizer 0x000000070a95b2c8, (a java.util.concurrent.locks.ReentrantLock$NonfairSync), which is held by "AuditTrailFilter.init""AuditTrailFilter.init": waiting to lock monitor 0x00007f4e4c824870 (object 0x000000070874b4f0, a hudson.ExtensionList$Lock), which is held by "GitLabConnection.migrate"Java stack information for the threads listed above: =================================================== "Handling GET /pbtest/ from 10.0.0.2 : Jetty (winstone)-14": at hudson.ExtensionList.ensureLoaded(ExtensionList.java:319) - waiting to lock <0x000000070874b4f0> (a hudson.ExtensionList$Lock) at hudson.ExtensionList.iterator(ExtensionList.java:172) at jenkins.util.HttpServletFilter$1.doFilter(HttpServletFilter.java:70) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:160) at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:166) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at jenkins.ErrorAttributeFilter.doFilter(ErrorAttributeFilter.java:29) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:160) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:170) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:53) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:86) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at jenkins.security.SuspiciousRequestFilter.doFilter(SuspiciousRequestFilter.java:38) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:569) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1580) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1553) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122) at org.eclipse.jetty.server.Server.handle(Server.java:563) at org.eclipse.jetty.server.HttpChannel$RequestDispatchable.dispatch(HttpChannel.java:1598) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:753) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:501) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:287) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:558) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:379) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:146) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100) at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:421) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:390) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:277) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.run(AdaptiveExecutionStrategy.java:199) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149) at java.lang.Thread.run(java.base@17.0.10/Unknown Source) "GitLabConnection.migrate": at jdk.internal.misc.Unsafe.park(java.base@17.0.10/Native Method) - parking to wait for <0x000000070a95b2c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(java.base@17.0.10/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@17.0.10/Unknown Source) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@17.0.10/Unknown Source) at java.util.concurrent.locks.ReentrantLock$Sync.lock(java.base@17.0.10/Unknown Source) at java.util.concurrent.locks.ReentrantLock.lock(java.base@17.0.10/Unknown Source) at com.google.inject.internal.CycleDetectingLock$CycleDetectingLockFactory$ReentrantCycleDetectingLock.lockOrDetectPotentialLocksCycle(CycleDetectingLock.java:170) at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:160) at hudson.ExtensionFinder$GuiceFinder$FaultTolerantScope$1.get(ExtensionFinder.java:445) at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:45) at com.google.inject.internal.InjectorImpl$1.get(InjectorImpl.java:1148) at hudson.ExtensionFinder$GuiceFinder._find(ExtensionFinder.java:403) at hudson.ExtensionFinder$GuiceFinder.find(ExtensionFinder.java:394) at hudson.ClassicPluginStrategy.findComponents(ClassicPluginStrategy.java:335) at hudson.ExtensionList.load(ExtensionList.java:384) at hudson.ExtensionList.ensureLoaded(ExtensionList.java:320) - locked <0x000000070874b4f0> (a hudson.ExtensionList$Lock) at hudson.ExtensionList.iterator(ExtensionList.java:172) at jenkins.model.Jenkins.getDescriptor(Jenkins.java:1649) at com.dabsquared.gitlabjenkins.connection.GitLabConnection.migrate(GitLabConnection.java:211) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.10/Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source) at java.lang.reflect.Method.invoke(java.base@17.0.10/Unknown Source) at hudson.init.TaskMethodFinder.invoke(TaskMethodFinder.java:109) at hudson.init.TaskMethodFinder$TaskImpl.run(TaskMethodFinder.java:185) at org.jvnet.hudson.reactor.Reactor.runTask(Reactor.java:305) at jenkins.model.Jenkins$5.runTask(Jenkins.java:1170) at org.jvnet.hudson.reactor.Reactor$2.run(Reactor.java:221) at org.jvnet.hudson.reactor.Reactor$Node.run(Reactor.java:120) at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.10/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.10/Unknown Source) at java.lang.Thread.run(java.base@17.0.10/Unknown Source) "AuditTrailFilter.init": at hudson.ExtensionList.ensureLoaded(ExtensionList.java:319) - waiting to lock <0x000000070874b4f0> (a hudson.ExtensionList$Lock) at hudson.ExtensionList.iterator(ExtensionList.java:172) at com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentialsInItemGroup(CredentialsProvider.java:386) at com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(CredentialsProvider.java:348) at hudson.plugins.audit_trail.ElasticSearchAuditLogger.getUsernamePasswordCredentials(ElasticSearchAuditLogger.java:187) at hudson.plugins.audit_trail.ElasticSearchAuditLogger.configure(ElasticSearchAuditLogger.java:144) at hudson.plugins.audit_trail.ElasticSearchAuditLogger.readResolve(ElasticSearchAuditLogger.java:106) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.10/Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source) at java.lang.reflect.Method.invoke(java.base@17.0.10/Unknown Source) at com.thoughtworks.xstream.core.util.SerializationMembers.callReadResolve(SerializationMembers.java:78) at hudson.util.RobustReflectionConverter.unmarshal(RobustReflectionConverter.java:290) at com.thoughtworks.xstream.core.TreeUnmarshaller.convert(TreeUnmarshaller.java:74) at com.thoughtworks.xstream.core.AbstractReferenceUnmarshaller.convert(AbstractReferenceUnmarshaller.java:72) at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:68) at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:52) at com.thoughtworks.xstream.converters.collections.AbstractCollectionConverter.readBareItem(AbstractCollectionConverter.java:132) at hudson.util.RobustCollectionConverter.populateCollection(RobustCollectionConverter.java:87) at com.thoughtworks.xstream.converters.collections.CollectionConverter.unmarshal(CollectionConverter.java:81) at hudson.util.RobustCollectionConverter.unmarshal(RobustCollectionConverter.java:78) at com.thoughtworks.xstream.core.TreeUnmarshaller.convert(TreeUnmarshaller.java:74) at com.thoughtworks.xstream.core.AbstractReferenceUnmarshaller.convert(AbstractReferenceUnmarshaller.java:72) at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:68) at hudson.util.RobustReflectionConverter.unmarshalField(RobustReflectionConverter.java:454) at hudson.util.RobustReflectionConverter.doUnmarshal(RobustReflectionConverter.java:350) at hudson.util.RobustReflectionConverter.unmarshal(RobustReflectionConverter.java:289) at com.thoughtworks.xstream.core.TreeUnmarshaller.convert(TreeUnmarshaller.java:74) at com.thoughtworks.xstream.core.AbstractReferenceUnmarshaller.convert(AbstractReferenceUnmarshaller.java:72) at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:68) at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:52) at com.thoughtworks.xstream.core.TreeUnmarshaller.start(TreeUnmarshaller.java:136) at com.thoughtworks.xstream.core.AbstractTreeMarshallingStrategy.unmarshal(AbstractTreeMarshallingStrategy.java:32) at com.thoughtworks.xstream.XStream.unmarshal(XStream.java:1464) at hudson.util.XStream2.unmarshal(XStream2.java:230) at hudson.util.XStream2.unmarshal(XStream2.java:201) at com.thoughtworks.xstream.XStream.unmarshal(XStream.java:1441) at hudson.XmlFile.unmarshal(XmlFile.java:196) at hudson.XmlFile.unmarshal(XmlFile.java:179) at hudson.model.Descriptor.load(Descriptor.java:935) - locked <0x000000070e407118> (a hudson.plugins.audit_trail.AuditTrailPlugin) at hudson.plugins.audit_trail.AuditTrailPlugin.<init>(AuditTrailPlugin.java:136) at hudson.plugins.audit_trail.AuditTrailPlugin$$FastClassByGuice$$5596c496.GUICE$TRAMPOLINE(<generated>) at hudson.plugins.audit_trail.AuditTrailPlugin$$FastClassByGuice$$5596c496.apply(<generated>) at com.google.inject.internal.DefaultConstructionProxyFactory$FastClassProxy.newInstance(DefaultConstructionProxyFactory.java:82) at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:114) at com.google.inject.internal.ConstructorInjector.access$000(ConstructorInjector.java:33) at com.google.inject.internal.ConstructorInjector$1.call(ConstructorInjector.java:98) at com.google.inject.internal.ProvisionListenerStackCallback$Provision.provision(ProvisionListenerStackCallback.java:109) at hudson.ExtensionFinder$GuiceFinder$SezpozModule.onProvision(ExtensionFinder.java:610) at com.google.inject.internal.ProvisionListenerStackCallback$Provision.provision(ProvisionListenerStackCallback.java:117) at com.google.inject.internal.ProvisionListenerStackCallback.provision(ProvisionListenerStackCallback.java:66) at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:93) at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:300) at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40) at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:169) at hudson.ExtensionFinder$GuiceFinder$FaultTolerantScope$1.get(ExtensionFinder.java:445) at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:45) at com.google.inject.internal.SingleFieldInjector.inject(SingleFieldInjector.java:50) at com.google.inject.internal.MembersInjectorImpl.injectMembers(MembersInjectorImpl.java:146) at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:124) at com.google.inject.internal.ConstructorInjector.access$000(ConstructorInjector.java:33) at com.google.inject.internal.ConstructorInjector$1.call(ConstructorInjector.java:98) at com.google.inject.internal.ProvisionListenerStackCallback$Provision.provision(ProvisionListenerStackCallback.java:109) at hudson.ExtensionFinder$GuiceFinder$SezpozModule.onProvision(ExtensionFinder.java:610) at com.google.inject.internal.ProvisionListenerStackCallback$Provision.provision(ProvisionListenerStackCallback.java:117) at com.google.inject.internal.ProvisionListenerStackCallback.provision(ProvisionListenerStackCallback.java:66) at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:93) at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:300) at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40) at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:169) at hudson.ExtensionFinder$GuiceFinder$FaultTolerantScope$1.get(ExtensionFinder.java:445) at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:45) at com.google.inject.internal.InjectorImpl$1.get(InjectorImpl.java:1148) at com.google.inject.internal.InjectorImpl.getInstance(InjectorImpl.java:1186) at jenkins.ProxyInjector.getInstance(ProxyInjector.java:113) at hudson.plugins.audit_trail.AuditTrailFilter.init(AuditTrailFilter.java:153) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.10/Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source) at java.lang.reflect.Method.invoke(java.base@17.0.10/Unknown Source) at hudson.init.TaskMethodFinder.invoke(TaskMethodFinder.java:109) at hudson.init.TaskMethodFinder$TaskImpl.run(TaskMethodFinder.java:185) at org.jvnet.hudson.reactor.Reactor.runTask(Reactor.java:305) at jenkins.model.Jenkins$5.runTask(Jenkins.java:1170) at org.jvnet.hudson.reactor.Reactor$2.run(Reactor.java:221) at org.jvnet.hudson.reactor.Reactor$Node.run(Reactor.java:120) at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68) at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.10/Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.10/Unknown Source) at java.lang.Thread.run(java.base@17.0.10/Unknown Source)Found 1 deadlock.