-
Type:
Bug
-
Resolution: Fixed
-
Priority:
Major
-
Component/s: audit-trail-plugin, gitlab-plugin
-
None
-
395.vce180b_359a_b_5
 Steps to Reproduce
- Install a brand new controller using our Jenkins job.
- Look for the plugin updates, install them manually, and click on the restart option at the bottom.
- If there are issues with plugins, then we will get into this situation of the Jenkins application being unreachable or down.Â
ound one Java-level deadlock: ============================= "Handling GET /pbtest/ from 10.0.0.2 : Jetty (winstone)-14":  waiting to lock monitor 0x00007f4e4c824870 (object 0x000000070874b4f0, a hudson.ExtensionList$Lock),  which is held by "GitLabConnection.migrate""GitLabConnection.migrate":  waiting for ownable synchronizer 0x000000070a95b2c8, (a java.util.concurrent.locks.ReentrantLock$NonfairSync),  which is held by "AuditTrailFilter.init""AuditTrailFilter.init":  waiting to lock monitor 0x00007f4e4c824870 (object 0x000000070874b4f0, a hudson.ExtensionList$Lock),  which is held by "GitLabConnection.migrate"Java stack information for the threads listed above: =================================================== "Handling GET /pbtest/ from 10.0.0.2 : Jetty (winstone)-14":     at hudson.ExtensionList.ensureLoaded(ExtensionList.java:319)     - waiting to lock <0x000000070874b4f0> (a hudson.ExtensionList$Lock)     at hudson.ExtensionList.iterator(ExtensionList.java:172)     at jenkins.util.HttpServletFilter$1.doFilter(HttpServletFilter.java:70)     at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:160)     at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:166)     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)     at jenkins.ErrorAttributeFilter.doFilter(ErrorAttributeFilter.java:29)     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)     at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:160)     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)     at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:170)     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)     at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:53)     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)     at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:86)     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)     at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30)     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)     at jenkins.security.SuspiciousRequestFilter.doFilter(SuspiciousRequestFilter.java:38)     at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)     at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)     at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)     at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)     at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:569)     at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)     at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)     at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1580)     at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)     at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384)     at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)     at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)     at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1553)     at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)     at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306)     at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)     at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)     at org.eclipse.jetty.server.Server.handle(Server.java:563)     at org.eclipse.jetty.server.HttpChannel$RequestDispatchable.dispatch(HttpChannel.java:1598)     at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:753)     at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:501)     at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:287)     at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)     at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)     at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:558)     at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:379)     at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:146)     at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)     at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)     at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:421)     at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:390)     at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:277)     at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.run(AdaptiveExecutionStrategy.java:199)     at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)     at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969)     at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194)     at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149)     at java.lang.Thread.run(java.base@17.0.10/Unknown Source) "GitLabConnection.migrate":     at jdk.internal.misc.Unsafe.park(java.base@17.0.10/Native Method)     - parking to wait for  <0x000000070a95b2c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync)     at java.util.concurrent.locks.LockSupport.park(java.base@17.0.10/Unknown Source)     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@17.0.10/Unknown Source)     at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@17.0.10/Unknown Source)     at java.util.concurrent.locks.ReentrantLock$Sync.lock(java.base@17.0.10/Unknown Source)     at java.util.concurrent.locks.ReentrantLock.lock(java.base@17.0.10/Unknown Source)     at com.google.inject.internal.CycleDetectingLock$CycleDetectingLockFactory$ReentrantCycleDetectingLock.lockOrDetectPotentialLocksCycle(CycleDetectingLock.java:170)     at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:160)     at hudson.ExtensionFinder$GuiceFinder$FaultTolerantScope$1.get(ExtensionFinder.java:445)     at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:45)     at com.google.inject.internal.InjectorImpl$1.get(InjectorImpl.java:1148)     at hudson.ExtensionFinder$GuiceFinder._find(ExtensionFinder.java:403)     at hudson.ExtensionFinder$GuiceFinder.find(ExtensionFinder.java:394)     at hudson.ClassicPluginStrategy.findComponents(ClassicPluginStrategy.java:335)     at hudson.ExtensionList.load(ExtensionList.java:384)     at hudson.ExtensionList.ensureLoaded(ExtensionList.java:320)     - locked <0x000000070874b4f0> (a hudson.ExtensionList$Lock)     at hudson.ExtensionList.iterator(ExtensionList.java:172)     at jenkins.model.Jenkins.getDescriptor(Jenkins.java:1649)     at com.dabsquared.gitlabjenkins.connection.GitLabConnection.migrate(GitLabConnection.java:211)     at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.10/Native Method)     at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source)     at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source)     at java.lang.reflect.Method.invoke(java.base@17.0.10/Unknown Source)     at hudson.init.TaskMethodFinder.invoke(TaskMethodFinder.java:109)     at hudson.init.TaskMethodFinder$TaskImpl.run(TaskMethodFinder.java:185)     at org.jvnet.hudson.reactor.Reactor.runTask(Reactor.java:305)     at jenkins.model.Jenkins$5.runTask(Jenkins.java:1170)     at org.jvnet.hudson.reactor.Reactor$2.run(Reactor.java:221)     at org.jvnet.hudson.reactor.Reactor$Node.run(Reactor.java:120)     at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68)     at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.10/Unknown Source)     at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.10/Unknown Source)     at java.lang.Thread.run(java.base@17.0.10/Unknown Source) "AuditTrailFilter.init":     at hudson.ExtensionList.ensureLoaded(ExtensionList.java:319)     - waiting to lock <0x000000070874b4f0> (a hudson.ExtensionList$Lock)     at hudson.ExtensionList.iterator(ExtensionList.java:172)     at com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentialsInItemGroup(CredentialsProvider.java:386)     at com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(CredentialsProvider.java:348)     at hudson.plugins.audit_trail.ElasticSearchAuditLogger.getUsernamePasswordCredentials(ElasticSearchAuditLogger.java:187)     at hudson.plugins.audit_trail.ElasticSearchAuditLogger.configure(ElasticSearchAuditLogger.java:144)     at hudson.plugins.audit_trail.ElasticSearchAuditLogger.readResolve(ElasticSearchAuditLogger.java:106)     at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.10/Native Method)     at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source)     at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source)     at java.lang.reflect.Method.invoke(java.base@17.0.10/Unknown Source)     at com.thoughtworks.xstream.core.util.SerializationMembers.callReadResolve(SerializationMembers.java:78)     at hudson.util.RobustReflectionConverter.unmarshal(RobustReflectionConverter.java:290)     at com.thoughtworks.xstream.core.TreeUnmarshaller.convert(TreeUnmarshaller.java:74)     at com.thoughtworks.xstream.core.AbstractReferenceUnmarshaller.convert(AbstractReferenceUnmarshaller.java:72)     at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:68)     at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:52)     at com.thoughtworks.xstream.converters.collections.AbstractCollectionConverter.readBareItem(AbstractCollectionConverter.java:132)     at hudson.util.RobustCollectionConverter.populateCollection(RobustCollectionConverter.java:87)     at com.thoughtworks.xstream.converters.collections.CollectionConverter.unmarshal(CollectionConverter.java:81)     at hudson.util.RobustCollectionConverter.unmarshal(RobustCollectionConverter.java:78)     at com.thoughtworks.xstream.core.TreeUnmarshaller.convert(TreeUnmarshaller.java:74)     at com.thoughtworks.xstream.core.AbstractReferenceUnmarshaller.convert(AbstractReferenceUnmarshaller.java:72)     at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:68)     at hudson.util.RobustReflectionConverter.unmarshalField(RobustReflectionConverter.java:454)     at hudson.util.RobustReflectionConverter.doUnmarshal(RobustReflectionConverter.java:350)     at hudson.util.RobustReflectionConverter.unmarshal(RobustReflectionConverter.java:289)     at com.thoughtworks.xstream.core.TreeUnmarshaller.convert(TreeUnmarshaller.java:74)     at com.thoughtworks.xstream.core.AbstractReferenceUnmarshaller.convert(AbstractReferenceUnmarshaller.java:72)     at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:68)     at com.thoughtworks.xstream.core.TreeUnmarshaller.convertAnother(TreeUnmarshaller.java:52)     at com.thoughtworks.xstream.core.TreeUnmarshaller.start(TreeUnmarshaller.java:136)     at com.thoughtworks.xstream.core.AbstractTreeMarshallingStrategy.unmarshal(AbstractTreeMarshallingStrategy.java:32)     at com.thoughtworks.xstream.XStream.unmarshal(XStream.java:1464)     at hudson.util.XStream2.unmarshal(XStream2.java:230)     at hudson.util.XStream2.unmarshal(XStream2.java:201)     at com.thoughtworks.xstream.XStream.unmarshal(XStream.java:1441)     at hudson.XmlFile.unmarshal(XmlFile.java:196)     at hudson.XmlFile.unmarshal(XmlFile.java:179)     at hudson.model.Descriptor.load(Descriptor.java:935)     - locked <0x000000070e407118> (a hudson.plugins.audit_trail.AuditTrailPlugin)     at hudson.plugins.audit_trail.AuditTrailPlugin.<init>(AuditTrailPlugin.java:136)     at hudson.plugins.audit_trail.AuditTrailPlugin$$FastClassByGuice$$5596c496.GUICE$TRAMPOLINE(<generated>)     at hudson.plugins.audit_trail.AuditTrailPlugin$$FastClassByGuice$$5596c496.apply(<generated>)     at com.google.inject.internal.DefaultConstructionProxyFactory$FastClassProxy.newInstance(DefaultConstructionProxyFactory.java:82)     at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:114)     at com.google.inject.internal.ConstructorInjector.access$000(ConstructorInjector.java:33)     at com.google.inject.internal.ConstructorInjector$1.call(ConstructorInjector.java:98)     at com.google.inject.internal.ProvisionListenerStackCallback$Provision.provision(ProvisionListenerStackCallback.java:109)     at hudson.ExtensionFinder$GuiceFinder$SezpozModule.onProvision(ExtensionFinder.java:610)     at com.google.inject.internal.ProvisionListenerStackCallback$Provision.provision(ProvisionListenerStackCallback.java:117)     at com.google.inject.internal.ProvisionListenerStackCallback.provision(ProvisionListenerStackCallback.java:66)     at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:93)     at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:300)     at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40)     at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:169)     at hudson.ExtensionFinder$GuiceFinder$FaultTolerantScope$1.get(ExtensionFinder.java:445)     at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:45)     at com.google.inject.internal.SingleFieldInjector.inject(SingleFieldInjector.java:50)     at com.google.inject.internal.MembersInjectorImpl.injectMembers(MembersInjectorImpl.java:146)     at com.google.inject.internal.ConstructorInjector.provision(ConstructorInjector.java:124)     at com.google.inject.internal.ConstructorInjector.access$000(ConstructorInjector.java:33)     at com.google.inject.internal.ConstructorInjector$1.call(ConstructorInjector.java:98)     at com.google.inject.internal.ProvisionListenerStackCallback$Provision.provision(ProvisionListenerStackCallback.java:109)     at hudson.ExtensionFinder$GuiceFinder$SezpozModule.onProvision(ExtensionFinder.java:610)     at com.google.inject.internal.ProvisionListenerStackCallback$Provision.provision(ProvisionListenerStackCallback.java:117)     at com.google.inject.internal.ProvisionListenerStackCallback.provision(ProvisionListenerStackCallback.java:66)     at com.google.inject.internal.ConstructorInjector.construct(ConstructorInjector.java:93)     at com.google.inject.internal.ConstructorBindingImpl$Factory.get(ConstructorBindingImpl.java:300)     at com.google.inject.internal.ProviderToInternalFactoryAdapter.get(ProviderToInternalFactoryAdapter.java:40)     at com.google.inject.internal.SingletonScope$1.get(SingletonScope.java:169)     at hudson.ExtensionFinder$GuiceFinder$FaultTolerantScope$1.get(ExtensionFinder.java:445)     at com.google.inject.internal.InternalFactoryToProviderAdapter.get(InternalFactoryToProviderAdapter.java:45)     at com.google.inject.internal.InjectorImpl$1.get(InjectorImpl.java:1148)     at com.google.inject.internal.InjectorImpl.getInstance(InjectorImpl.java:1186)     at jenkins.ProxyInjector.getInstance(ProxyInjector.java:113)     at hudson.plugins.audit_trail.AuditTrailFilter.init(AuditTrailFilter.java:153)     at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(java.base@17.0.10/Native Method)     at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source)     at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(java.base@17.0.10/Unknown Source)     at java.lang.reflect.Method.invoke(java.base@17.0.10/Unknown Source)     at hudson.init.TaskMethodFinder.invoke(TaskMethodFinder.java:109)     at hudson.init.TaskMethodFinder$TaskImpl.run(TaskMethodFinder.java:185)     at org.jvnet.hudson.reactor.Reactor.runTask(Reactor.java:305)     at jenkins.model.Jenkins$5.runTask(Jenkins.java:1170)     at org.jvnet.hudson.reactor.Reactor$2.run(Reactor.java:221)     at org.jvnet.hudson.reactor.Reactor$Node.run(Reactor.java:120)     at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68)     at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@17.0.10/Unknown Source)     at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@17.0.10/Unknown Source)     at java.lang.Thread.run(java.base@17.0.10/Unknown Source)Found 1 deadlock.