After Hudson as been running for an hour or so, we start getting 500 errors on all top level view and management pages, including any login pages. Individual project status pages still work, however.

Stack trace is as follows (also at http://hudson.pastebin.com/6ULMBK9G ):
HTTP Status 500 -

type Exception report

message

description The server encountered an internal error () that prevented it from fulfilling this request.

exception

javax.servlet.ServletException: org.apache.commons.jelly.JellyTagException: jar:file:/opt/nas/ci/hudson/webapps/hudson/WEB-INF/lib/hudson-core-1.377.jar!/lib/hudson/executors.jelly:67:24: <j:otherwise> java.lang.AbstractMethodError
org.kohsuke.stapler.jelly.JellyClassTearOff.serveIndexJelly(JellyClassTearOff.java:48)
org.kohsuke.stapler.jelly.JellyFacet.handleIndexRequest(JellyFacet.java:83)
org.kohsuke.stapler.Stapler.invoke(Stapler.java:523)
org.kohsuke.stapler.Stapler.invoke(Stapler.java:578)
org.kohsuke.stapler.Stapler.invoke(Stapler.java:450)
org.kohsuke.stapler.Stapler.service(Stapler.java:132)
javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:94)
hudson.plugins.greenballs.GreenBallFilter.doFilter(GreenBallFilter.java:51)
hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:97)
hudson.plugins.audit_trail.AuditTrailFilter.doFilter(AuditTrailFilter.java:64)
hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:97)
hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:86)
hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:47)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:84)
hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:51)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:166)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.providers.anonymous.AnonymousProcessingFilter.doFilter(AnonymousProcessingFilter.java:125)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.rememberme.RememberMeProcessingFilter.doFilter(RememberMeProcessingFilter.java:142)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.AbstractProcessingFilter.doFilter(AbstractProcessingFilter.java:271)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.basicauth.BasicProcessingFilter.doFilter(BasicProcessingFilter.java:173)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.context.HttpSessionContextIntegrationFilter.doFilter(HttpSessionContextIntegrationFilter.java:249)
hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:66)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:76)
hudson.security.HudsonFilter.doFilter(HudsonFilter.java:164)
root cause

org.apache.commons.jelly.JellyTagException: jar:file:/opt/nas/ci/hudson/webapps/hudson/WEB-INF/lib/hudson-core-1.377.jar!/lib/hudson/executors.jelly:67:24: <j:otherwise> java.lang.AbstractMethodError
org.apache.commons.jelly.impl.TagScript.handleException(TagScript.java:728)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:290)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.TagSupport.invokeBody(TagSupport.java:161)
org.apache.commons.jelly.tags.core.ChooseTag.doTag(ChooseTag.java:38)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.DynamicTag.doTag(DynamicTag.java:81)
org.apache.commons.jelly.impl.StaticTagScript.run(StaticTagScript.java:121)
org.apache.commons.jelly.TagSupport.invokeBody(TagSupport.java:161)
org.apache.commons.jelly.tags.core.ForEachTag.doTag(ForEachTag.java:150)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.TagSupport.invokeBody(TagSupport.java:161)
org.apache.commons.jelly.tags.core.ForEachTag.doTag(ForEachTag.java:150)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:75)
org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:96)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:96)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:75)
org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.tags.core.CoreTagLibrary$1.run(CoreTagLibrary.java:98)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:96)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.IncludeTag.doTag(IncludeTag.java:118)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:75)
org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:96)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.CompressTag.doTag(CompressTag.java:21)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.kohsuke.stapler.jelly.DefaultScriptInvoker.invokeScript(DefaultScriptInvoker.java:40)
org.kohsuke.stapler.jelly.DefaultScriptInvoker.invokeScript(DefaultScriptInvoker.java:30)
org.kohsuke.stapler.jelly.JellyClassTearOff.serveIndexJelly(JellyClassTearOff.java:43)
org.kohsuke.stapler.jelly.JellyFacet.handleIndexRequest(JellyFacet.java:83)
org.kohsuke.stapler.Stapler.invoke(Stapler.java:523)
org.kohsuke.stapler.Stapler.invoke(Stapler.java:578)
org.kohsuke.stapler.Stapler.invoke(Stapler.java:450)
org.kohsuke.stapler.Stapler.service(Stapler.java:132)
javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:94)
hudson.plugins.greenballs.GreenBallFilter.doFilter(GreenBallFilter.java:51)
hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:97)
hudson.plugins.audit_trail.AuditTrailFilter.doFilter(AuditTrailFilter.java:64)
hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:97)
hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:86)
hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:47)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:84)
hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:51)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:166)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.providers.anonymous.AnonymousProcessingFilter.doFilter(AnonymousProcessingFilter.java:125)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.rememberme.RememberMeProcessingFilter.doFilter(RememberMeProcessingFilter.java:142)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.AbstractProcessingFilter.doFilter(AbstractProcessingFilter.java:271)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.basicauth.BasicProcessingFilter.doFilter(BasicProcessingFilter.java:173)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.context.HttpSessionContextIntegrationFilter.doFilter(HttpSessionContextIntegrationFilter.java:249)
hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:66)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:76)
hudson.security.HudsonFilter.doFilter(HudsonFilter.java:164)
root cause

java.lang.AbstractMethodError
hudson.model.Executor.hasStopPermission(Executor.java:366)
sun.reflect.GeneratedMethodAccessor155.invoke(Unknown Source)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
java.lang.reflect.Method.invoke(Method.java:597)
org.apache.commons.jexl.util.introspection.UberspectImpl$VelMethodImpl.invoke(UberspectImpl.java:258)
org.apache.commons.jexl.parser.ASTMethod.execute(ASTMethod.java:104)
org.apache.commons.jexl.parser.ASTReference.execute(ASTReference.java:83)
org.apache.commons.jexl.parser.ASTReference.value(ASTReference.java:57)
org.apache.commons.jexl.parser.ASTReferenceExpression.value(ASTReferenceExpression.java:51)
org.apache.commons.jexl.ExpressionImpl.evaluate(ExpressionImpl.java:80)
hudson.ExpressionFactory2$JexlExpression.evaluate(ExpressionFactory2.java:72)
org.apache.commons.jelly.expression.ExpressionSupport.evaluateRecurse(ExpressionSupport.java:61)
org.apache.commons.jelly.expression.ExpressionSupport.evaluateAsBoolean(ExpressionSupport.java:71)
org.apache.commons.jelly.tags.core.CoreTagLibrary$1.run(CoreTagLibrary.java:97)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.TagSupport.invokeBody(TagSupport.java:161)
org.apache.commons.jelly.tags.core.OtherwiseTag.doTag(OtherwiseTag.java:41)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.TagSupport.invokeBody(TagSupport.java:161)
org.apache.commons.jelly.tags.core.ChooseTag.doTag(ChooseTag.java:38)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.DynamicTag.doTag(DynamicTag.java:81)
org.apache.commons.jelly.impl.StaticTagScript.run(StaticTagScript.java:121)
org.apache.commons.jelly.TagSupport.invokeBody(TagSupport.java:161)
org.apache.commons.jelly.tags.core.ForEachTag.doTag(ForEachTag.java:150)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.TagSupport.invokeBody(TagSupport.java:161)
org.apache.commons.jelly.tags.core.ForEachTag.doTag(ForEachTag.java:150)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:75)
org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:96)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:96)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:75)
org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.tags.core.CoreTagLibrary$1.run(CoreTagLibrary.java:98)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:96)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.IncludeTag.doTag(IncludeTag.java:118)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.CallTagLibScript$1.run(CallTagLibScript.java:75)
org.apache.commons.jelly.tags.define.InvokeBodyTag.doTag(InvokeBodyTag.java:91)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.ReallyStaticTagLibrary$1.run(ReallyStaticTagLibrary.java:76)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.apache.commons.jelly.tags.core.CoreTagLibrary$2.run(CoreTagLibrary.java:105)
org.kohsuke.stapler.jelly.CallTagLibScript.run(CallTagLibScript.java:96)
org.apache.commons.jelly.impl.ScriptBlock.run(ScriptBlock.java:95)
org.kohsuke.stapler.jelly.CompressTag.doTag(CompressTag.java:21)
org.apache.commons.jelly.impl.TagScript.run(TagScript.java:270)
org.kohsuke.stapler.jelly.DefaultScriptInvoker.invokeScript(DefaultScriptInvoker.java:40)
org.kohsuke.stapler.jelly.DefaultScriptInvoker.invokeScript(DefaultScriptInvoker.java:30)
org.kohsuke.stapler.jelly.JellyClassTearOff.serveIndexJelly(JellyClassTearOff.java:43)
org.kohsuke.stapler.jelly.JellyFacet.handleIndexRequest(JellyFacet.java:83)
org.kohsuke.stapler.Stapler.invoke(Stapler.java:523)
org.kohsuke.stapler.Stapler.invoke(Stapler.java:578)
org.kohsuke.stapler.Stapler.invoke(Stapler.java:450)
org.kohsuke.stapler.Stapler.service(Stapler.java:132)
javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:94)
hudson.plugins.greenballs.GreenBallFilter.doFilter(GreenBallFilter.java:51)
hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:97)
hudson.plugins.audit_trail.AuditTrailFilter.doFilter(AuditTrailFilter.java:64)
hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:97)
hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:86)
hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:47)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:84)
hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:51)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:166)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.providers.anonymous.AnonymousProcessingFilter.doFilter(AnonymousProcessingFilter.java:125)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.rememberme.RememberMeProcessingFilter.doFilter(RememberMeProcessingFilter.java:142)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.AbstractProcessingFilter.doFilter(AbstractProcessingFilter.java:271)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.ui.basicauth.BasicProcessingFilter.doFilter(BasicProcessingFilter.java:173)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
org.acegisecurity.context.HttpSessionContextIntegrationFilter.doFilter(HttpSessionContextIntegrationFilter.java:249)
hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:66)
hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87)
hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:76)
hudson.security.HudsonFilter.doFilter(HudsonFilter.java:164)
note The full stack trace of the root cause is available in the Apache Tomcat/6.0.18 logs.

Apache Tomcat/6.0.18