-
Bug
-
Resolution: Fixed
-
Blocker
-
None
-
Jenkins ver. 2.150.1
OS: Linux RHEL 7.5
-
Powered by SuggestiMate
Cannot clear workspace:
java.nio.file.FileSystemException: c:\jenkins2x\workspace\QA\ILAB-K1-Functional-Tests\k1-footnotes\k1-georgia-footnotes: The process cannot access the file because it is being used by another process.
Problem exists only on Windows slaves!
[JENKINS-55567] Windows cannot delete job workspace
Full STACK TRACE:
Stack trace
java.nio.file.FileSystemException: c:\jenkins2x\workspace\QA\ILAB-K1-Functional-Tests\k1-footnotes\k1-georgia-footnotes: The process cannot access the file because it is being used by another process. at sun.nio.fs.WindowsException.translateToIOException(Unknown Source) at sun.nio.fs.WindowsException.rethrowAsIOException(Unknown Source) at sun.nio.fs.WindowsException.rethrowAsIOException(Unknown Source) at sun.nio.fs.WindowsFileSystemProvider.implDelete(Unknown Source) at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(Unknown Source) at java.nio.file.Files.deleteIfExists(Unknown Source) at hudson.Util.tryOnceDeleteFile(Util.java:316) at hudson.Util.deleteFile(Util.java:272) Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to JNLP4-connect connection from rc88af.testbbh.com/10.115.92.10:62280 at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741) at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357) at hudson.remoting.Channel.call(Channel.java:955) at hudson.FilePath.act(FilePath.java:1072) at hudson.FilePath.act(FilePath.java:1061) at hudson.FilePath.deleteRecursive(FilePath.java:1268) at hudson.model.AbstractProject.doDoWipeOutWorkspace(AbstractProject.java:1870) at java.lang.invoke.MethodHandle.invokeWithArguments(MethodHandle.java:627) at org.kohsuke.stapler.Function$MethodFunction.invoke(Function.java:396) at org.kohsuke.stapler.Function$InstanceFunction.invoke(Function.java:408) at org.kohsuke.stapler.interceptor.RequirePOST$Processor.invoke(RequirePOST.java:77) at org.kohsuke.stapler.PreInvokeInterceptedFunction.invoke(PreInvokeInterceptedFunction.java:26) at org.kohsuke.stapler.Function.bindAndInvoke(Function.java:212) at org.kohsuke.stapler.Function.bindAndInvokeAndServeResponse(Function.java:145) at org.kohsuke.stapler.MetaClass$11.doDispatch(MetaClass.java:537) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:739) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:870) at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:282) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:739) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:870) at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:282) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:739) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:870) at org.kohsuke.stapler.MetaClass$4.doDispatch(MetaClass.java:282) at org.kohsuke.stapler.NameBasedDispatcher.dispatch(NameBasedDispatcher.java:58) at org.kohsuke.stapler.Stapler.tryInvoke(Stapler.java:739) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:870) at org.kohsuke.stapler.Stapler.invoke(Stapler.java:668) at org.kohsuke.stapler.Stapler.service(Stapler.java:238) at javax.servlet.http.HttpServlet.service(HttpServlet.java:790) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:865) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1655) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:154) at org.jenkinsci.plugins.ssegateway.Endpoint$SSEListenChannelFilter.doFilter(Endpoint.java:243) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151) at io.jenkins.blueocean.ResourceCacheControl.doFilter(ResourceCacheControl.java:134) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151) at io.jenkins.blueocean.auth.jwt.impl.JwtAuthenticationFilter.doFilter(JwtAuthenticationFilter.java:61) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151) at com.smartcodeltd.jenkinsci.plugin.assetbundler.filters.LessCSS.doFilter(LessCSS.java:47) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151) at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:239) at net.bull.javamelody.MonitoringFilter.doFilter(MonitoringFilter.java:215) at net.bull.javamelody.PluginMonitoringFilter.doFilter(PluginMonitoringFilter.java:88) at org.jvnet.hudson.plugins.monitoring.HudsonMonitoringFilter.doFilter(HudsonMonitoringFilter.java:114) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151) at hudson.plugins.greenballs.GreenBallFilter.doFilter(GreenBallFilter.java:59) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151) at hudson.plugins.scm_sync_configuration.extensions.ScmSyncConfigurationFilter$1.call(ScmSyncConfigurationFilter.java:49) at hudson.plugins.scm_sync_configuration.extensions.ScmSyncConfigurationFilter$1.call(ScmSyncConfigurationFilter.java:44) at hudson.plugins.scm_sync_configuration.ScmSyncConfigurationDataProvider.provideRequestDuring(ScmSyncConfigurationDataProvider.java:106) at hudson.plugins.scm_sync_configuration.extensions.ScmSyncConfigurationFilter.doFilter(ScmSyncConfigurationFilter.java:44) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151) at jenkins.telemetry.impl.UserLanguages$AcceptLanguageFilter.doFilter(UserLanguages.java:128) at hudson.util.PluginServletFilter$1.doFilter(PluginServletFilter.java:151) at hudson.util.PluginServletFilter.doFilter(PluginServletFilter.java:157) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at hudson.security.csrf.CrumbFilter.doFilter(CrumbFilter.java:99) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:84) at hudson.security.UnwrapSecurityExceptionFilter.doFilter(UnwrapSecurityExceptionFilter.java:51) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at jenkins.security.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:117) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at org.acegisecurity.providers.anonymous.AnonymousProcessingFilter.doFilter(AnonymousProcessingFilter.java:125) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at org.acegisecurity.ui.rememberme.RememberMeProcessingFilter.doFilter(RememberMeProcessingFilter.java:142) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at org.acegisecurity.ui.AbstractProcessingFilter.doFilter(AbstractProcessingFilter.java:271) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at jenkins.security.BasicHeaderProcessor.doFilter(BasicHeaderProcessor.java:93) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at org.acegisecurity.context.HttpSessionContextIntegrationFilter.doFilter(HttpSessionContextIntegrationFilter.java:249) at hudson.security.HttpSessionContextIntegrationFilter2.doFilter(HttpSessionContextIntegrationFilter2.java:67) at hudson.security.ChainedServletFilter$1.doFilter(ChainedServletFilter.java:87) at hudson.security.ChainedServletFilter.doFilter(ChainedServletFilter.java:90) at hudson.security.HudsonFilter.doFilter(HudsonFilter.java:171) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.kohsuke.stapler.compression.CompressionFilter.doFilter(CompressionFilter.java:49) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at hudson.util.CharacterEncodingFilter.doFilter(CharacterEncodingFilter.java:82) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.kohsuke.stapler.DiagnosticThreadNameFilter.doFilter(DiagnosticThreadNameFilter.java:30) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1642) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:146) at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:524) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:257) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1340) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1242) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:503) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:364) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:260) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:411) at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:305) at org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:159) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:118) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:333) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:310) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:168) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:126) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:366) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:765) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:683) at java.lang.Thread.run(Thread.java:748) Caused: java.io.IOException: Unable to delete 'c:\jenkins2x\workspace\QA\ILAB-K1-Functional-Tests\k1-footnotes\k1-georgia-footnotes'. Tried 3 times (of a maximum of 3) waiting 0.1 sec between attempts. at hudson.Util.deleteFile(Util.java:277) at hudson.FilePath.deleteRecursive(FilePath.java:1305) at hudson.FilePath.deleteContentsRecursive(FilePath.java:1314) at hudson.FilePath.deleteRecursive(FilePath.java:1296) at hudson.FilePath.deleteContentsRecursive(FilePath.java:1314) at hudson.FilePath.deleteRecursive(FilePath.java:1296) at hudson.FilePath.access$1600(FilePath.java:213) at hudson.FilePath$DeleteRecursive.invoke(FilePath.java:1274) at hudson.FilePath$DeleteRecursive.invoke(FilePath.java:1270) at hudson.FilePath$FileCallableWrapper.call(FilePath.java:3086) at hudson.remoting.UserRequest.perform(UserRequest.java:212) at hudson.remoting.UserRequest.perform(UserRequest.java:54) at hudson.remoting.Request$2.run(Request.java:369) at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72) at java.util.concurrent.FutureTask.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at hudson.remoting.Engine$1.lambda$newThread$0(Engine.java:93) at java.lang.Thread.run(Unknown Source)
The Windows file system will not allow open files to be deleted. That intentional file system design is made visible to programs with failures like this. The challenge is to determine which specific item is keeping files open on Windows after the job completes.
If the problem repeats consistently with the current configuration and if the problem did not consistently repeat in a previous configuration, you can attempt to downgrade individual plugins and use that as a way to assign the issue to a specific plugin. Usually you'll want to try downgrading the plugins which are installed less frequently first and choose plugins which are installed less frequently later. Plugins that are used in many installations are less likely to inadvertently leave files open.
Please provide specific steps which will allow someone else to duplicate the problem. I don't see this problem among the many jobs that I run on Windows machines.
my job was just clone git repository using git plugin and jgit option for Windows - that's all.
Also Windows was not updated or any change was not implemented on system.
It was working before normally but after some update Java is using git files constantly and does not allow clear workspace.
There is no such problem on Linux nodes.
I just downgraded Script Security to 1.49 but lots of dependencies failed but Wipe out current workspace works currently. But still this is not a solution.
Dependency errors:
Some plugins could not be loaded due to unsatisfied dependencies. Fix these issues and restart Jenkins to restore the functionality provided by these plugins.
Pipeline: Groovy version 2.61.1Script Security Plugin v1.49 is older than required. To fix, install v1.50 or later.
Downstream dependency errors:
These plugins failed to load because of one or more of the errors above. Fix those and these plugins will load again.
GitHub Pipeline for Blue Ocean version 1.10.1Pipeline implementation for Blue Ocean v1.10.1 failed to load. Fix this plugin first.Pipeline: Declarative Extension Points API version 1.3.4.1Pipeline: Groovy v2.61.1 failed to load. Fix this plugin first.Git Pipeline for Blue Ocean version 1.10.1Pipeline implementation for Blue Ocean v1.10.1 failed to load. Fix this plugin first.Pipeline Graph Analysis Plugin version 1.9Pipeline: Groovy v2.61.1 failed to load. Fix this plugin first.Pipeline version 2.6Pipeline: Stage View Plugin v2.10 failed to load. Fix this plugin first.Pipeline: Stage View Plugin version 2.10Pipeline: REST API Plugin v2.10 failed to load. Fix this plugin first.Micro Focus Application Automation Tools version 5.6.2Pipeline: Groovy v2.61.1 failed to load. Fix this plugin first.Pipeline SCM API for Blue Ocean version 1.10.1Pipeline: Multibranch v2.20 failed to load. Fix this plugin first.Pipeline: Declarative version 1.3.4.1Pipeline: Shared Groovy Libraries v2.12 failed to load. Fix this plugin first.Pipeline: Shared Groovy Libraries version 2.12Pipeline: Groovy v2.61.1 failed to load. Fix this plugin first.Bitbucket Pipeline for Blue Ocean version 1.10.1Pipeline implementation for Blue Ocean v1.10.1 failed to load. Fix this plugin first.Blue Ocean version 1.10.1Bitbucket Pipeline for Blue Ocean v1.10.1 failed to load. Fix this plugin first.Pipeline implementation for Blue Ocean version 1.10.1Pipeline SCM API for Blue Ocean v1.10.1 failed to load. Fix this plugin first.qTest Plugin version 1.4.0Pipeline v2.6 failed to load. Fix this plugin first.Pipeline: Multibranch version 2.20Pipeline: Groovy v2.61.1 failed to load. Fix this plugin first.Docker Pipeline version 1.17Pipeline: Groovy v2.61.1 failed to load. Fix this plugin first.Pipeline: REST API Plugin version 2.10Pipeline Graph Analysis Plugin v1.9 failed to load. Fix this plugin first.Display URL for Blue Ocean version 2.2.0Pipeline: Multibranch v2.20 failed to load. Fix this plugin first.Events API for Blue Ocean version 1.10.1Pipeline implementation for Blue Ocean v1.10.1 failed to load. Fix this plugin first.Blue Ocean Pipeline Editor version 1.10.1Pipeline implementation for Blue Ocean v1.10.1 failed to load. Fix this plugin first.Pipeline: Declarative Agent API version 1.1.1Pipeline: Declarative Extension Points API v1.3.4.1 failed to load. Fix this plugin first.
So I assume one of these is blocking wipe out operation.
If your Windows agent can install command line git, you can test if the problem is related to JGit or is something else. Change the job definition to use command line git instead of JGit. If the problem disappears when using command line git and reappears when using JGit, then the file is being left open by a call to JGit in the git client plugin. That doesn't ultimately decide which plugin is causing the issue, but it does more insight of areas to check.
If the problem does not disappear, then it is not likely in JGit. Since the JGit version has not changed recently, I would be surprised if this were specifically related to JGit.
I'm unable to duplicate the problem. I've created the JENKINS-55567 test which performs a deleteDir() before cloning a test repository on Windows with JGit, then performs a deleteDir() after cloning the test repository. No errors were reported on the several different agents which executed that test.
Please provide detailed steps that will allow others to see the problem.
We have very similar issue with maven type jobs on windows. Quick head dump analysis showed files are opened by jgit components.
Here is list of plugins being updated.
UPD: upgrading workflow-job to 2.31 causes such behavior with pipeline type job, but we didn't done any heap dump analysis, so it could be something different.
EDIT: it seems it is not the same issue, at least no word about jgit in this dump.
Recently upgraded plugins:
rw-rr-. 1 jenkins jenkins 777781 Jan 7 08:25 blueocean-jwt.jpirw-rr-. 1 jenkins jenkins 380523 Jan 7 08:25 workflow-support.jpirw-rr-. 1 jenkins jenkins 103117 Jan 7 08:26 blueocean-rest.jpirw-rr-. 1 jenkins jenkins 471206 Jan 7 08:27 cloudbees-bitbucket-branch-source.jpirw-rr-. 1 jenkins jenkins 32508 Jan 7 08:28 sidebar-link.jpirw-rr-. 1 jenkins jenkins 2361669 Jan 7 08:28 jenkins-design-language.jpirw-rr-. 1 jenkins jenkins 1068997 Jan 7 08:28 blueocean-core-js.jpirw-rr-. 1 jenkins jenkins 302313 Jan 7 08:28 blueocean-pipeline-scm-api.jpirw-rr-. 1 jenkins jenkins 2745810 Jan 7 08:32 blueocean-dashboard.jpirw-rr-. 1 jenkins jenkins 69224 Jan 7 08:33 blueocean-github-pipeline.jpirw-rr-. 1 jenkins jenkins 707751 Jan 7 08:33 blueocean-personalization.jpirw-rr-. 1 jenkins jenkins 1410526 Jan 7 08:35 blueocean-web.jpirw-rr-. 1 jenkins jenkins 84812 Jan 7 08:36 blueocean-commons.jpirw-rr-. 1 jenkins jenkins 183574 Jan 7 08:38 blueocean-pipeline-api-impl.jpirw-rr-. 1 jenkins jenkins 61281 Jan 7 08:40 blueocean-config.jpirw-rr-. 1 jenkins jenkins 18370944 Jan 7 08:41 analysis-model-api.jpirw-rr-. 1 jenkins jenkins 24870623 Jan 7 08:41 warnings-ng.jpirw-rr-. 1 jenkins jenkins 15529258 Jan 7 08:41 warnings.jpirw-rr-. 1 jenkins jenkins 788112 Jan 7 08:42 blueocean-git-pipeline.jpirw-rr-. 1 jenkins jenkins 2953593 Jan 7 08:43 job-dsl.jpirw-rr-. 1 jenkins jenkins 7110275 Jan 7 08:43 maven-artifact-choicelistprovider.jpirw-rr-. 1 jenkins jenkins 471045 Jan 7 08:51 windows-slaves.jpirw-rr-. 1 jenkins jenkins 38896 Jan 7 08:51 command-launcher.jpirw-rr-. 1 jenkins jenkins 7408336 Jan 7 08:51 findbugs.jpirw-rr-. 1 jenkins jenkins 13524 Jan 7 08:51 blueocean-jira.jpirw-rr-. 1 jenkins jenkins 2401159 Jan 7 08:51 blueocean-bitbucket-pipeline.jpirw-rr-. 1 jenkins jenkins 5124369 Jan 7 08:51 blueocean-events.jpirw-rr-. 1 jenkins jenkins 14790 Jan 7 08:51 blueocean-i18n.jpirw-rr-. 1 jenkins jenkins 144075 Jan 7 08:51 fitnesse.jpirw-rr-. 1 jenkins jenkins 106842 Jan 7 08:52 ssh-slaves.jpirw-rr-. 1 jenkins jenkins 44589 Jan 7 08:53 sse-gateway.jpirw-rr-. 1 jenkins jenkins 1065978 Jan 7 08:55 blueocean-rest-impl.jpirw-rr-. 1 jenkins jenkins 7363 Jan 7 08:55 blueocean.jpirw-rr-. 1 jenkins jenkins 1883676 Jan 7 08:55 blueocean-pipeline-editor.jpirw-rr-. 1 jenkins jenkins 3868734 Jan 7 08:58 xunit.jpirw-rr-. 1 jenkins jenkins 2599000 Jan 7 08:58 git-client.jpirw-rr-. 1 jenkins jenkins 113231 Jan 7 08:59 mailer.jpirw-rr-. 1 jenkins jenkins 43504 Jan 7 08:59 ansicolor.jpirw-rr-. 1 jenkins jenkins 137935 Jan 7 08:59 jenkins-multijob-plugin.jpirw-rr-. 1 jenkins jenkins 2418023 Jan 8 09:20 pipeline-model-api.jpi