Creating a template using a Groovy transformer (in this case actually a folder template inside a parent folder, with two job templates created inside it) was remarkably slow:
"Handling POST /job/.../createItem : http-8089-6" Id=5527 Group=main RUNNABLE
at java.util.zip.ZipFile.getEntry(Native Method)
at java.util.zip.ZipFile.getEntry(Unknown Source)
- locked java.util.jar.JarFile@1951d8d7
at java.util.jar.JarFile.getEntry(Unknown Source)
at java.util.jar.JarFile.getJarEntry(Unknown Source)
at org.apache.catalina.loader.WebappClassLoader.findResources(WebappClassLoader.java:1315)
- locked [Ljava.util.jar.JarFile;@7b9a0ac
at java.lang.ClassLoader.getResources(Unknown Source)
at java.lang.ClassLoader.getResources(Unknown Source)
at java.lang.ClassLoader.getResources(Unknown Source)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:957)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:935)
at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at jenkins.ClassLoaderReflectionToolkit.findResources(ClassLoaderReflectionToolkit.java:60)
at hudson.ClassicPluginStrategy$DependencyClassLoader.findResources(ClassicPluginStrategy.java:594)
at java.lang.ClassLoader.getResources(Unknown Source)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:957)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:935)
at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at jenkins.ClassLoaderReflectionToolkit.findResources(ClassLoaderReflectionToolkit.java:60)
at hudson.ClassicPluginStrategy$DependencyClassLoader.findResources(ClassicPluginStrategy.java:594)
at java.lang.ClassLoader.getResources(Unknown Source)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:957)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:935)
at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at jenkins.ClassLoaderReflectionToolkit.findResources(ClassLoaderReflectionToolkit.java:60)
at hudson.ClassicPluginStrategy$DependencyClassLoader.findResources(ClassicPluginStrategy.java:594)
at java.lang.ClassLoader.getResources(Unknown Source)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:957)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:935)
at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at jenkins.ClassLoaderReflectionToolkit.findResources(ClassLoaderReflectionToolkit.java:60)
at hudson.ClassicPluginStrategy$DependencyClassLoader.findResources(ClassicPluginStrategy.java:594)
at java.lang.ClassLoader.getResources(Unknown Source)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:957)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:935)
at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at jenkins.ClassLoaderReflectionToolkit.findResources(ClassLoaderReflectionToolkit.java:60)
at hudson.ClassicPluginStrategy$DependencyClassLoader.findResources(ClassicPluginStrategy.java:594)
at java.lang.ClassLoader.getResources(Unknown Source)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:957)
at hudson.PluginFirstClassLoader.findResources(PluginFirstClassLoader.java:79)
at org.apache.tools.ant.AntClassLoader.findResources(AntClassLoader.java:935)
at hudson.PluginFirstClassLoader.findResources(PluginFirstClassLoader.java:87)
at sun.reflect.GeneratedMethodAccessor24.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.lang.reflect.Method.invoke(Unknown Source)
at jenkins.ClassLoaderReflectionToolkit.findResources(ClassLoaderReflectionToolkit.java:60)
at hudson.PluginManager$UberClassLoader.findResources(PluginManager.java:1016)
at java.lang.ClassLoader.getResources(Unknown Source)
at java.lang.ClassLoader.getResources(Unknown Source)
at org.codehaus.groovy.transform.ASTTransformationVisitor.doAddGlobalTransforms(ASTTransformationVisitor.java:194)
at org.codehaus.groovy.transform.ASTTransformationVisitor.addGlobalTransforms(ASTTransformationVisitor.java:187)
at org.codehaus.groovy.transform.ASTTransformationVisitor.addPhaseOperations(ASTTransformationVisitor.java:150)
at org.codehaus.groovy.control.CompilationUnit.<init>(CompilationUnit.java:186)
at org.codehaus.groovy.control.CompilationUnit.<init>(CompilationUnit.java:118)
at groovy.lang.GroovyClassLoader.createCompilationUnit(GroovyClassLoader.java:476)
at groovy.lang.GroovyClassLoader.doParseClass(GroovyClassLoader.java:290)
at groovy.lang.GroovyClassLoader.parseClass(GroovyClassLoader.java:281)
- locked java.util.HashMap@70971d3
at groovy.lang.GroovyShell.parseClass(GroovyShell.java:731)
at groovy.lang.GroovyShell.parse(GroovyShell.java:743)
at groovy.lang.GroovyShell.parse(GroovyShell.java:770)
at com.cloudbees.hudson.plugins.modeling.impl.SimpleTemplateEngine.parseScript(SimpleTemplateEngine.java:134)
at com.cloudbees.hudson.plugins.modeling.impl.SimpleTemplateEngine.createTemplate(SimpleTemplateEngine.java:122)
at com.cloudbees.hudson.plugins.modeling.impl.SimpleTemplateEngine.createTemplate(SimpleTemplateEngine.java:116)
at groovy.text.TemplateEngine.createTemplate(TemplateEngine.java:38)
at com.cloudbees.hudson.plugins.modeling.transformer.GroovyTemplateModelTransformer.compile(GroovyTemplateModelTransformer.java:83)
at com.cloudbees.hudson.plugins.modeling.transformer.GroovyTemplateModelTransformer.compileTemplate(GroovyTemplateModelTransformer.java:75)
at com.cloudbees.hudson.plugins.modeling.transformer.GroovyTemplateModelTransformer.transform(GroovyTemplateModelTransformer.java:97)
at com.cloudbees.hudson.plugins.modeling.impl.entity.EntityInstance.produce(EntityInstance.java:149)
at com.cloudbees.hudson.plugins.modeling.impl.entity.EntityInstance.produce(EntityInstance.java:121)
at com.cloudbees.hudson.plugins.modeling.impl.jobTemplate.JobTemplateDescriptor.newInstance(JobTemplateDescriptor.java:47)
at hudson.model.ItemGroupMixIn.createProject(ItemGroupMixIn.java:259)
- locked com.cloudbees.hudson.plugins.folder.Folder$MixInImpl@37ee014f
at com.cloudbees.hudson.plugins.folder.Folder.createProject(Folder.java:666)
at com.cloudbees.hudson.plugins.folder.Folder.createProject(Folder.java:662)
at com.cloudbees.hudson.plugins.modeling.impl.folder.activities.CreateItemActivity.perform(CreateItemActivity.java:42)
at com.cloudbees.hudson.plugins.modeling.impl.folder.FolderTemplateDescriptor.newInstance(FolderTemplateDescriptor.java:54)
at com.cloudbees.hudson.plugins.modeling.impl.folder.FolderTemplateDescriptor.newInstance(FolderTemplateDescriptor.java:22)
at hudson.model.ItemGroupMixIn.createProject(ItemGroupMixIn.java:259)
- locked com.cloudbees.hudson.plugins.folder.Folder$MixInImpl@22c07d34
at hudson.model.ItemGroupMixIn.createTopLevelItem(ItemGroupMixIn.java:175)
- locked com.cloudbees.hudson.plugins.folder.Folder$MixInImpl@22c07d34
at com.cloudbees.hudson.plugins.folder.Folder.doCreateItem(Folder.java:597)
at com.cloudbees.hudson.plugins.folder.Folder.doCreateItem(Folder.java:122)
UberClassLoader.findResources calls findResources on each plugin, which would be fine, except apparently some of these calls involve deep recursions into plugin dependencies, and probably a lot of overlapping calls to findResources in the root web application. I am guessing this all turns out to be vastly slower than just checking each plugin ZIP file, plus the parent, once.
We are seeing very similar stacktraces in threads serving job configuration pages (and random jenkins-cli commands). Job configuration can take 30-45 seconds with about 80 plugins versus 2 seconds with no plugins. Jenkins uses 100% CPU while the page is loaded with very little IO. Every time I do a jstack, the offending thread's stacktrace looks very similar to the stacktrace you posted.