Uploaded image for project: 'Jenkins'
  1. Jenkins
  2. JENKINS-58711

"Container does not exist" error using "sh" in a declarative pipeline

      Attempting to use the sh command from inside a container block results in an error of the form java.io.IOException: container [name] does not exist in pod [id]. This same setup works OK when using the scripted pipeline.

      Example:

      pipeline {
        agent {
          kubernetes {
            yaml """
      apiVersion: v1
      kind: Pod
      spec:
        containers:
        - name: golang
          image: busybox
          command:
          - cat
          tty: true
      """
          }
        }
        options {
          skipDefaultCheckout true
        }
        stages {
          stage('Build') {
            steps {
              container('golang') {
                sh 'ls'
              }
            }
          }
        }
      }
      
      

      Running this pipeline yields:

      java.io.IOException: container [golang] does not exist in pod [default-vhhcf]
      	at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.waitUntilPodContainersAreReady(ContainerExecDecorator.java:476)
      	at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.doLaunch(ContainerExecDecorator.java:271)
      	at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.launch(ContainerExecDecorator.java:252)
      	at hudson.Launcher$ProcStarter.start(Launcher.java:455)
      	at org.jenkinsci.plugins.durabletask.BourneShellScript.launchWithCookie(BourneShellScript.java:206)
      	at org.jenkinsci.plugins.durabletask.FileMonitoringTask.launch(FileMonitoringTask.java:99)
      	at org.jenkinsci.plugins.workflow.steps.durable_task.DurableTaskStep$Execution.start(DurableTaskStep.java:302)
      	at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:268)
      	at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:176)
      	at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:122)
      	at groovy.lang.MetaClassImpl.invokeMethodOnGroovyObject(MetaClassImpl.java:1278)
      	at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1172)
      	at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1022)
      	at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:42)
      	at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:48)
      	at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:113)
      	at com.cloudbees.groovy.cps.sandbox.DefaultInvoker.methodCall(DefaultInvoker.java:20)
      	at getRepoTags.call(/var/jenkins_home/jobs/attr-dict-import/branches/master/builds/36/libs/jenkins/vars/getRepoTags.groovy:6)
      	at ___cps.transform___(Native Method)
      	at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:57)
      	at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:109)
      	at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:82)
      	at sun.reflect.GeneratedMethodAccessor380.invoke(Unknown Source)
      	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      	at java.lang.reflect.Method.invoke(Method.java:498)
      	at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
      	at com.cloudbees.groovy.cps.impl.CollectionLiteralBlock$ContinuationImpl.dispatch(CollectionLiteralBlock.java:55)
      	at com.cloudbees.groovy.cps.impl.CollectionLiteralBlock$ContinuationImpl.item(CollectionLiteralBlock.java:45)
      	at sun.reflect.GeneratedMethodAccessor276.invoke(Unknown Source)
      	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      	at java.lang.reflect.Method.invoke(Method.java:498)
      	at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
      	at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:103)
      	at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:82)
      	at sun.reflect.GeneratedMethodAccessor380.invoke(Unknown Source)
      	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      	at java.lang.reflect.Method.invoke(Method.java:498)
      	at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
      	at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:60)
      	at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:109)
      	at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:82)
      	at sun.reflect.GeneratedMethodAccessor380.invoke(Unknown Source)
      	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
      	at java.lang.reflect.Method.invoke(Method.java:498)
      	at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
      	at com.cloudbees.groovy.cps.impl.ConstantBlock.eval(ConstantBlock.java:21)
      	at com.cloudbees.groovy.cps.Next.step(Next.java:83)
      	at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:174)
      	at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:163)
      	at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:129)
      	at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:268)
      	at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:163)
      	at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$001(SandboxContinuable.java:18)
      	at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:51)
      	at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:174)
      	at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:347)
      	at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.access$200(CpsThreadGroup.java:93)
      	at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:259)
      	at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:247)
      	at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:64)
      	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      	at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:131)
      	at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
      	at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:59)
      	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
      	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
      	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
      	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
      	at java.lang.Thread.run(Thread.java:748)

          [JENKINS-58711] "Container does not exist" error using "sh" in a declarative pipeline

          Jack Whelpton added a comment - - edited

          First edit: added a sample implementation of "pull" to reproduce the problem, which I've narrowed down to an "sh" invocation in the shared library function.

          Second edit: actually, the shared library is not significant: the same error can be reproduced just using an "sh" command insider a "container" block in a declarative pipeline.

          Jack Whelpton added a comment - - edited First edit: added a sample implementation of "pull" to reproduce the problem, which I've narrowed down to an "sh" invocation in the shared library function. Second edit: actually, the shared library is not significant: the same error can be reproduced just using an "sh" command insider a "container" block in a declarative pipeline.

          Jack Whelpton added a comment -

          I can also reproduce the same error by running the declarative example from the Kubernetes plugin documentation as-is:

          https://github.com/jenkinsci/kubernetes-plugin/blob/master/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/declarative.groovy

          java.io.IOException: container [maven] does not exist in pod [default-jwg9h]
          	at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.waitUntilPodContainersAreReady(ContainerExecDecorator.java:476)
          	at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.doLaunch(ContainerExecDecorator.java:271)
          	at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.launch(ContainerExecDecorator.java:252)

          Jack Whelpton added a comment - I can also reproduce the same error by running the declarative example from the Kubernetes plugin documentation as-is: https://github.com/jenkinsci/kubernetes-plugin/blob/master/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/declarative.groovy java.io.IOException: container [maven] does not exist in pod [default-jwg9h] at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.waitUntilPodContainersAreReady(ContainerExecDecorator.java:476) at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.doLaunch(ContainerExecDecorator.java:271) at org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1.launch(ContainerExecDecorator.java:252)

          Jack Whelpton added a comment - - edited

          Last update for me today: I can also reproduce this by omitting the container {} block but adding defaultContainer 'golang' to the kubernetes{} block.

          Added Jenkins and k8s plugin versions to the Environment section.

          Jack Whelpton added a comment - - edited Last update for me today: I can also reproduce this by omitting the container {} block but adding defaultContainer 'golang' to the kubernetes{} block. Added Jenkins and k8s plugin versions to the Environment section.

          This issue seems to affect the kubernetes plugin version 1.12.4 as well.

          Panos Koumantos added a comment - This issue seems to affect the kubernetes plugin version 1.12.4 as well.

          This behavior is observed in Kubernetes plugin v1.25.7. The occurrence is not consistent. 

          We have 2 containers specified in the pod definition using cloudprovider.yml

          Out of 2 containers specified in the pod template (cloudprovider.yml) one of the containers (container that is not specified in the defaultContainer) were not starting when the job is triggered. 

          Here is the agent definition section in the Jenkinsfile

               agent {
                  kubernetes {
                      label "run-test"
                      defaultContainer 'container1'
                      yamlFile 'cloudprovider.yml'
                  }
              } 
          
              stages {
                 stage ('Build') { 
                      steps {
                          retry(2) {
                              mvn("version")
                          }
                      }
                  }
                  stage ('Build2') {
                      steps {
                          container('container2') {               
                              script {
                                  sh "ls -lah"
                              }
                          }
                      }
                  }

          In the above snippet, the container 2 sometimes fails to start and Build2 stage fails with "container [] does not exist in pod []"

          In the kubernetes label, if a random label is assigned (as shown in below snippet), both the pods are starting all the time

                  kubernetes {
                      label "run-test-${UUID.randomUUID().toString()}"
                      defaultContainer 'container1'
                      yamlFile 'cloudprovider.yml'
                  } 

          So, I suspect some where the random name assignment is misbehaving.

          Shankar Ramasamy added a comment - This behavior is observed in Kubernetes plugin v1.25.7. The occurrence is not consistent.  We have 2 containers specified in the pod definition using cloudprovider.yml Out of 2 containers specified in the pod template (cloudprovider.yml) one of the containers (container that is not specified in the defaultContainer) were not starting when the job is triggered.  Here is the agent definition section in the Jenkinsfile agent { kubernetes { label "run-test" defaultContainer 'container1' yamlFile 'cloudprovider.yml' } } stages { stage ( 'Build' ) { steps { retry(2) { mvn( "version" ) } } } stage ( 'Build2' ) { steps { container( 'container2' ) { script { sh "ls -lah" } } } } In the above snippet, the container 2 sometimes fails to start and Build2 stage fails with "container [] does not exist in pod []" In the kubernetes label, if a random label is assigned (as shown in below snippet), both the pods are starting all the time kubernetes { label "run-test-${UUID.randomUUID().toString()}" defaultContainer 'container1' yamlFile 'cloudprovider.yml' } So, I suspect some where the random name assignment is misbehaving.

          Don't use the label attribute and let the plugin allocate a generated label.

          Vincent Latombe added a comment - Don't use the label attribute and let the plugin allocate a generated label.

          Kohsuke Kawaguchi added a comment - - edited

          I'm reopening this issue because I see the same problem reported by Jack the original reporter. Note that he didn't use the label attribute at all. That is an extra complication created by Shanker, who may be possibly reporting a separate issue. The problem also reproduces consistently for me.

          I can see from the console log that the pod getting launched does not reflect any of the YAML configuration I added, which would naturally result in "container not found". The YAML file is getting parsed as if I intentionally introduce a syntax error, that gets detected. But if I introduce intentional semantic error (say add `bogus:bogus`) that does not get flagged.

          This error is particularly frustraing because, as also stated by Jack, the example copy-pasted straight from the top of the plugin documentation doesn't even work, and while I'm sure there's something I'm doing wrong, there's no indication of what to look to solve this problem, so I'm left clueless:

          pipeline {
            agent {
              kubernetes {
                yaml """\
                  apiVersion: v1
                  kind: Pod
                  metadata:
                  labels:
                      some-label: some-label-value
                  spec:
                  containers:
                  - name: maven
                      image: maven:alpine
                      command:
                      - cat
                      tty: true
                  - name: busybox
                      image: busybox
                      command:
                      - cat
                      tty: true
                  """.stripIndent()
              }
            }
            stages {
              stage('Run maven') {
                steps {
                  container('maven') {
                    sh 'mvn -version'
                  }
                  container('busybox') {
                    sh '/bin/busybox'
                  }
                }
              }
            }
          }
          

          Kohsuke Kawaguchi added a comment - - edited I'm reopening this issue because I see the same problem reported by Jack the original reporter. Note that he didn't use the label attribute at all. That is an extra complication created by Shanker, who may be possibly reporting a separate issue. The problem also reproduces consistently for me. I can see from the console log that the pod getting launched does not reflect any of the YAML configuration I added, which would naturally result in "container not found". The YAML file is getting parsed as if I intentionally introduce a syntax error, that gets detected. But if I introduce intentional semantic error (say add `bogus:bogus`) that does not get flagged. This error is particularly frustraing because, as also stated by Jack, the example copy-pasted straight from the top of the plugin documentation doesn't even work, and while I'm sure there's something I'm doing wrong, there's no indication of what to look to solve this problem, so I'm left clueless: pipeline { agent { kubernetes { yaml """\ apiVersion: v1 kind: Pod metadata: labels: some-label: some-label-value spec: containers: - name: maven image: maven:alpine command: - cat tty: true - name: busybox image: busybox command: - cat tty: true """.stripIndent() } } stages { stage( 'Run maven' ) { steps { container( 'maven' ) { sh 'mvn -version' } container( 'busybox' ) { sh '/bin/busybox' } } } } }

          is your indentation correct or a copy & paste problem? you are missing spaces before labels and containers

                  apiVersion: v1
                  kind: Pod
                  metadata:
                      labels:
                          some-label: some-label-value
                  spec:
                      containers:
          

          Carlos Sanchez added a comment - is your indentation correct or a copy & paste problem? you are missing spaces before labels and containers apiVersion: v1 kind: Pod metadata: labels: some-label: some-label-value spec: containers:

          Seems like a recent PR introduced an indentation problem in the readme. Fixed.

          Vincent Latombe added a comment - Seems like a recent PR introduced an indentation problem in the readme. Fixed .

            vlatombe Vincent Latombe
            jackwhelpton Jack Whelpton
            Votes:
            1 Vote for this issue
            Watchers:
            7 Start watching this issue

              Created:
              Updated: