Uploaded image for project: 'Jenkins'
  1. Jenkins
  2. JENKINS-64904

kubernetes agent: sometimes, java.lang.OutOfMemoryError: unable to create new native thread

XMLWordPrintable

    • Icon: Improvement Improvement
    • Resolution: Unresolved
    • Icon: Major Major
    • kubernetes-plugin
    • None
    • jenkins 2.249.2 (container), kubernetes plugin: 1.27.7
      also jenkins 2.263.4, kubernetes plugin: latest

       

      Sometimes, I got this error.

      after agent provisioning on kubernetes, the first command (sh) is failed.  open files, max user process on jenkins master, kubernetes worker node seems to be irrelevant. Also, physical memory is enough.

      I want to know whether it is master issue or agent(node) issue.

       
      08:27:52 Running on user-perf0216-indigo-pig-14-4n-14-0219081356-14-2mn01-xnd-q3zpj in /home/jenkins/agent/workspace/user-perf0216-indigo-pig-14/4n-14-0219081356
      08:27:52 [Pipeline] {
      08:27:52 [Pipeline] timeout
      08:27:52 Timeout set to expire in 15 min
      08:27:52 [Pipeline] {
      08:27:52 [Pipeline] stage
      08:27:52 [Pipeline]

      { (checkout) 08:27:52 [Pipeline] sh 08:27:53 [Pipeline] }

      08:27:53 [Pipeline] // stage
      08:27:53 [Pipeline] }
      08:27:53 [Pipeline] // timeout
      08:27:53 [Pipeline] }
      08:27:53 [Pipeline] // node
      08:27:53 [Pipeline] }
      08:27:53 [Pipeline] // podTemplate
      08:27:53 [Pipeline] }
      08:27:53 [Pipeline] // withEnv
      08:27:53 [Pipeline] End of Pipeline
      08:27:53 Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to JNLP4-connect connection from 133.186.216.214/133.186.216.214:21111
      08:27:53 at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1800)
      08:27:53 at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
      08:27:53 at hudson.remoting.Channel.call(Channel.java:1001)
      08:27:53 at hudson.Launcher$RemoteLauncher.launch(Launcher.java:1121)
      08:27:53 at hudson.Launcher$ProcStarter.start(Launcher.java:508)
      08:27:53 at org.jenkinsci.plugins.durabletask.BourneShellScript.launchWithCookie(BourneShellScript.java:234)
      08:27:53 at org.jenkinsci.plugins.durabletask.FileMonitoringTask.launch(FileMonitoringTask.java:103)
      08:27:53 at org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep$Execution.start(DurableTaskStep.java:320)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:286)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:179)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:122)
      08:27:53 at sun.reflect.GeneratedMethodAccessor215.invoke(Unknown Source)
      08:27:53 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      08:27:53 at java.lang.reflect.Method.invoke(Method.java:498)
      08:27:53 at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93)
      08:27:53 at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325)
      08:27:53 at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1213)
      08:27:53 at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1022)
      08:27:53 at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:42)
      08:27:53 at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:48)
      08:27:53 at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:113)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:163)
      08:27:53 at org.kohsuke.groovy.sandbox.GroovyInterceptor.onMethodCall(GroovyInterceptor.java:23)
      08:27:53 at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onMethodCall(SandboxInterceptor.java:157)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:161)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:165)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.methodCall(SandboxInvoker.java:17)
      08:27:53 at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:86)
      08:27:53 at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:113)
      08:27:53 at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:83)
      08:27:53 at sun.reflect.GeneratedMethodAccessor164.invoke(Unknown Source)
      08:27:53 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      08:27:53 at java.lang.reflect.Method.invoke(Method.java:498)
      08:27:53 at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
      08:27:53 at com.cloudbees.groovy.cps.impl.ConstantBlock.eval(ConstantBlock.java:21)
      08:27:53 at com.cloudbees.groovy.cps.Next.step(Next.java:83)
      08:27:53 at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:174)
      08:27:53 at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:163)
      08:27:53 at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:129)
      08:27:53 at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:268)
      08:27:53 at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:163)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$001(SandboxContinuable.java:18)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:51)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:185)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:400)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.access$400(CpsThreadGroup.java:96)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:312)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:276)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:67)
      08:27:53 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      08:27:53 at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:136)
      08:27:53 at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
      08:27:53 at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:59)
      08:27:53 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
      08:27:53 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      08:27:53 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
      08:27:53 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
      08:27:53 java.lang.OutOfMemoryError: unable to create new native thread
      08:27:53 at java.lang.Thread.start0(Native Method)
      08:27:53 at java.lang.Thread.start(Thread.java:717)
      08:27:53 at java.util.concurrent.ThreadPoolExecutor.addWorker(ThreadPoolExecutor.java:957)
      08:27:53 at java.util.concurrent.ThreadPoolExecutor.ensurePrestart(ThreadPoolExecutor.java:1603)
      08:27:53 at java.util.concurrent.ScheduledThreadPoolExecutor.delayedExecute(ScheduledThreadPoolExecutor.java:334)
      08:27:53 at java.util.concurrent.ScheduledThreadPoolExecutor.schedule(ScheduledThreadPoolExecutor.java:533)
      08:27:53 at jenkins.util.InterceptingScheduledExecutorService.schedule(InterceptingScheduledExecutorService.java:49)
      08:27:53 at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream.reschedule(DelayBufferedOutputStream.java:72)
      08:27:53 at org.jenkinsci.plugins.workflow.log.DelayBufferedOutputStream.<init>(DelayBufferedOutputStream.java:68)
      08:27:53 at org.jenkinsci.plugins.workflow.log.BufferedBuildListener$Replacement.readResolve(BufferedBuildListener.java:77)
      08:27:53 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
      08:27:53 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
      08:27:53 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      08:27:53 at java.lang.reflect.Method.invoke(Method.java:498)
      08:27:53 at java.io.ObjectStreamClass.invokeReadResolve(ObjectStreamClass.java:1260)
      08:27:53 at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2133)
      08:27:53 at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1625)
      08:27:53 at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2342)
      08:27:53 at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2266)
      08:27:53 at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2124)
      08:27:53 at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1625)
      08:27:53 at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:2342)
      08:27:53 at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:2266)
      08:27:53 at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2124)
      08:27:53 at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1625)
      08:27:53 at java.io.ObjectInputStream.readObject(ObjectInputStream.java:465)
      08:27:53 at java.io.ObjectInputStream.readObject(ObjectInputStream.java:423)
      08:27:53 at hudson.remoting.UserRequest.deserialize(UserRequest.java:290)
      08:27:53 at hudson.remoting.UserRequest.perform(UserRequest.java:189)
      08:27:53 at hudson.remoting.UserRequest.perform(UserRequest.java:54)
      08:27:53 at hudson.remoting.Request$2.run(Request.java:369)
      08:27:53 at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
      08:27:53 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      08:27:53 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
      08:27:53 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
      08:27:53 at hudson.remoting.Engine$1.lambda$newThread$0(Engine.java:117)
      08:27:53 Caused: java.io.IOException: Remote call on JNLP4-connect connection from 133.186.216.214/133.186.216.214:21111 failed
      08:27:53 at hudson.remoting.Channel.call(Channel.java:1007)
      08:27:53 at hudson.Launcher$RemoteLauncher.launch(Launcher.java:1121)
      08:27:53 at hudson.Launcher$ProcStarter.start(Launcher.java:508)
      08:27:53 at org.jenkinsci.plugins.durabletask.BourneShellScript.launchWithCookie(BourneShellScript.java:234)
      08:27:53 at org.jenkinsci.plugins.durabletask.FileMonitoringTask.launch(FileMonitoringTask.java:103)
      08:27:53 at org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep$Execution.start(DurableTaskStep.java:320)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:286)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:179)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:122)
      08:27:53 at sun.reflect.GeneratedMethodAccessor215.invoke(Unknown Source)
      08:27:53 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      08:27:53 at java.lang.reflect.Method.invoke(Method.java:498)
      08:27:53 at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93)
      08:27:53 at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325)
      08:27:53 at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1213)
      08:27:53 at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1022)
      08:27:53 at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:42)
      08:27:53 at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:48)
      08:27:53 at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:113)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:163)
      08:27:53 at org.kohsuke.groovy.sandbox.GroovyInterceptor.onMethodCall(GroovyInterceptor.java:23)
      08:27:53 at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onMethodCall(SandboxInterceptor.java:157)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:161)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:165)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:135)
      08:27:53 at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.methodCall(SandboxInvoker.java:17)
      08:27:53 at WorkflowScript.run(WorkflowScript:49)
      08:27:53 at __cps.transform__(Native Method)
      08:27:53 at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:86)
      08:27:53 at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:113)
      08:27:53 at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:83)
      08:27:53 at sun.reflect.GeneratedMethodAccessor164.invoke(Unknown Source)
      08:27:53 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      08:27:53 at java.lang.reflect.Method.invoke(Method.java:498)
      08:27:53 at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
      08:27:53 at com.cloudbees.groovy.cps.impl.ConstantBlock.eval(ConstantBlock.java:21)
      08:27:53 at com.cloudbees.groovy.cps.Next.step(Next.java:83)
      08:27:53 at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:174)
      08:27:53 at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:163)
      08:27:53 at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:129)
      08:27:53 at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:268)
      08:27:53 at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:163)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$001(SandboxContinuable.java:18)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:51)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:185)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:400)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.access$400(CpsThreadGroup.java:96)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:312)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:276)
      08:27:53 at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:67)
      08:27:53 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      08:27:53 at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:136)
      08:27:53 at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
      08:27:53 at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:59)
      08:27:53 at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
      08:27:53 at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      08:27:53 at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
      08:27:53 at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
      08:27:53 at java.lang.Thread.run(Thread.java:748)
      08:27:53 Finished: FAILURE
       

            Unassigned Unassigned
            linugee hyunil shin
            Votes:
            1 Vote for this issue
            Watchers:
            6 Start watching this issue

              Created:
              Updated: