package hudson.scm;
import hudson.FilePath;
import hudson.Launcher;
import hudson.Util;
import hudson.FilePath.FileCallable;
import hudson.model.AbstractBuild;
import hudson.model.AbstractProject;
import hudson.model.BuildListener;
import hudson.model.Hudson;
import hudson.model.TaskListener;
import hudson.remoting.Callable;
import hudson.remoting.Channel;
import hudson.remoting.VirtualChannel;
import hudson.triggers.SCMTrigger;
import hudson.util.EditDistance;
import hudson.util.FormFieldValidator;
import hudson.util.IOException2;
import hudson.util.MultipartFormDataParser;
import hudson.util.Scrambler;
import hudson.util.StreamCopyThread;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.Serializable;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.StringTokenizer;
import java.util.Map.Entry;
import java.util.logging.ConsoleHandler;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Pattern;
import javax.servlet.ServletException;
import javax.xml.transform.stream.StreamResult;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.io.FileUtils;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.taskdefs.Chmod;
import org.kohsuke.stapler.StaplerRequest;
import org.kohsuke.stapler.StaplerResponse;
import org.tmatesoft.svn.core.SVNDirEntry;
import org.tmatesoft.svn.core.SVNErrorCode;
import org.tmatesoft.svn.core.SVNErrorMessage;
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNNodeKind;
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationProvider;
import org.tmatesoft.svn.core.auth.SVNAuthentication;
import org.tmatesoft.svn.core.auth.SVNPasswordAuthentication;
import org.tmatesoft.svn.core.auth.SVNSSHAuthentication;
import org.tmatesoft.svn.core.auth.SVNSSLAuthentication;
import org.tmatesoft.svn.core.auth.SVNUserNameAuthentication;
import org.tmatesoft.svn.core.internal.io.dav.DAVRepositoryFactory;
import org.tmatesoft.svn.core.internal.io.fs.FSRepositoryFactory;
import org.tmatesoft.svn.core.internal.io.svn.SVNRepositoryFactoryImpl;
import org.tmatesoft.svn.core.internal.util.SVNPathUtil;
import org.tmatesoft.svn.core.internal.wc.DefaultSVNAuthenticationManager;
import org.tmatesoft.svn.core.internal.wc.SVNErrorManager;
import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.wc.SVNClientManager;
import org.tmatesoft.svn.core.wc.SVNInfo;
import org.tmatesoft.svn.core.wc.SVNRevision;
import org.tmatesoft.svn.core.wc.SVNUpdateClient;
import org.tmatesoft.svn.core.wc.SVNWCClient;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
import ch.ethz.ssh2.SCPClient;
/**
* Subversion SCM.
*
*
* Because this instance refers to some other classes that are not necessarily
* Java serializable (like {@link #browser}), remotable {@link FileCallable}s all
* need to be declared as static inner classes.
*
* @author Kohsuke Kawaguchi
*/
public class SubversionSCM extends SCM implements Serializable {
/**
* the locations field is used to store all configured SVN locations (with
* their local and remote part). Direct access to this filed should be
* avoided and the getLocations() method should be used instead. This is
* needed to make importing of old hudson-configurations possible as
* getLocations() will check if the modules field has been set and import
* the data.
*
* @since 1.91
*/
private ModuleLocation[] locations = new ModuleLocation[0];
private boolean useUpdate;
private final SubversionRepositoryBrowser browser;
// No longer in use but left for serialization compatibility.
@Deprecated
private String modules;
public SubversionSCM(String[] remoteLocations, String[] localLocations,
boolean useUpdate, SubversionRepositoryBrowser browser) {
List modules = new ArrayList();
if (remoteLocations != null && localLocations != null) {
int entries = Math.min(remoteLocations.length, localLocations.length);
for (int i = 0; i < entries; i++) {
// the remote (repository) location
String remoteLoc = nullify(remoteLocations[i]);
if (remoteLoc != null) {// null if skipped
remoteLoc = Util.removeTrailingSlash(remoteLoc.trim());
modules.add(new ModuleLocation(remoteLoc, nullify(localLocations[i])));
}
}
}
locations = modules.toArray(new ModuleLocation[modules.size()]);
this.useUpdate = useUpdate;
this.browser = browser;
}
/**
* @deprecated
* as of 1.91. Use {@link #getLocations()} instead.
*/
public String getModules() {
return null;
}
/**
* list of all configured svn locations
*
* @since 1.91
*/
public ModuleLocation[] getLocations() {
// check if we've got a old location
if (modules != null) {
// import the old configuration
List oldLocations = new ArrayList();
StringTokenizer tokens = new StringTokenizer(modules);
while (tokens.hasMoreTokens()) {
// the remote (repository location)
// the normalized name is always without the trailing '/'
String remoteLoc = Util.removeTrailingSlash(tokens.nextToken());
oldLocations.add(new ModuleLocation(remoteLoc, null));
}
locations = oldLocations.toArray(new ModuleLocation[oldLocations.size()]);
modules = null;
}
return locations;
}
public boolean isUseUpdate() {
return useUpdate;
}
@Override
public SubversionRepositoryBrowser getBrowser() {
return browser;
}
/**
* Sets the SVN_REVISION environment variable during the build.
*/
@Override
public void buildEnvVars(AbstractBuild build, Map env) {
super.buildEnvVars(build, env);
try {
Map revisions = parseRevisionFile(build);
if(locations.length==1) {
Long rev = revisions.get(locations[0].remote);
if(rev!=null)
env.put("SVN_REVISION",rev.toString());
}
// it's not clear what to do if there are more than one modules.
// if we always return locations[0].remote, it'll be difficult
// to change this later (to something more sensible, such as
// choosing the "root module" or whatever), so let's not set
// anything for now.
} catch (IOException e) {
// ignore this error
}
}
/**
* Called after checkout/update has finished to compute the changelog.
*/
private boolean calcChangeLog(AbstractBuild,?> build, File changelogFile, BuildListener listener, List externals) throws IOException, InterruptedException {
if(build.getPreviousBuild()==null) {
// nothing to compare against
return createEmptyChangeLog(changelogFile, listener, "log");
}
// some users reported that the file gets created with size 0. I suspect
// maybe some XSLT engine doesn't close the stream properly.
// so let's do it by ourselves to be really sure that the stream gets closed.
OutputStream os = new BufferedOutputStream(new FileOutputStream(changelogFile));
boolean created;
try {
created = new SubversionChangeLogBuilder(build, listener, this).run(externals, new StreamResult(os));
} finally {
os.close();
}
if(!created)
createEmptyChangeLog(changelogFile, listener, "log");
return true;
}
/**
* Reads the revision file of the specified build.
*
* @return
* map from {@link SvnInfo#url Subversion URL} to its revision.
*/
/*package*/ static Map parseRevisionFile(AbstractBuild build) throws IOException {
Map revisions = new HashMap(); // module -> revision
{// read the revision file of the last build
File file = getRevisionFile(build);
if(!file.exists())
// nothing to compare against
return revisions;
BufferedReader br = new BufferedReader(new FileReader(file));
try {
String line;
while((line=br.readLine())!=null) {
int index = line.lastIndexOf('/');
if(index<0) {
continue; // invalid line?
}
try {
revisions.put(line.substring(0,index), Long.parseLong(line.substring(index+1)));
} catch (NumberFormatException e) {
// perhaps a corrupted line. ignore
}
}
} finally {
br.close();
}
}
return revisions;
}
/**
* Parses the file that stores the locations in the workspace where modules loaded by svn:external
* is placed.
*/
/*package*/ static List parseExternalsFile(AbstractProject project) throws IOException {
List ext = new ArrayList(); // workspace-relative path
{// read the revision file of the last build
File file = getExternalsFile(project);
if(!file.exists())
// nothing to compare against
return ext;
BufferedReader br = new BufferedReader(new FileReader(file));
try {
String line;
while((line=br.readLine())!=null) {
ext.add(line);
}
} finally {
br.close();
}
}
return ext;
}
public boolean checkout(AbstractBuild build, Launcher launcher, FilePath workspace, final BuildListener listener, File changelogFile) throws IOException, InterruptedException {
List externals = checkout(build,workspace,listener);
if(externals==null)
return false;
// write out the revision file
PrintWriter w = new PrintWriter(new FileOutputStream(getRevisionFile(build)));
try {
Map revMap = workspace.act(new BuildRevisionMapTask(this, listener, externals));
for (Entry e : revMap.entrySet()) {
w.println( e.getKey() +'/'+ e.getValue().revision );
}
build.addAction(new SubversionTagAction(build,revMap.values()));
} finally {
w.close();
}
// write out the externals info
w = new PrintWriter(new FileOutputStream(getExternalsFile(build.getProject())));
try {
for (String p : externals) {
w.println( p );
}
} finally {
w.close();
}
return calcChangeLog(build, changelogFile, listener, externals);
}
/**
* Performs the checkout or update, depending on the configuration and workspace state.
*
*
* Use canonical path to avoid SVNKit/symlink problem as described in
* https://wiki.svnkit.com/SVNKit_FAQ
*
* @return null
* if the operation failed. Otherwise the set of local workspace paths
* (relative to the workspace root) that has loaded due to svn:external.
*/
private List checkout(AbstractBuild build, FilePath workspace, TaskListener listener) throws IOException, InterruptedException {
try {
if (! repositoryLocationsExist()) {
// Disable this project, see issue #763
listener.getLogger().println("One or more repository locations do not exist anymore for " + build.getProject().getName() + ", project will be disabled.");
build.getProject().makeDisabled(true);
return null;
}
} catch (SVNException e) {
e.printStackTrace(listener.error(e.getMessage()));
return null;
}
Boolean isUpdatable = useUpdate && workspace.act(new IsUpdatableTask(this, listener));
return workspace.act(new CheckOutTask(this, build.getTimestamp().getTime(), isUpdatable, listener));
}
private static class CheckOutTask implements FileCallable> {
private final ISVNAuthenticationProvider authProvider;
private final Date timestamp;
// true to "svn update", false to "svn checkout".
private final boolean update;
private final TaskListener listener;
private final ModuleLocation[] locations;
public CheckOutTask(SubversionSCM parent, Date timestamp, boolean update, TaskListener listener) {
this.authProvider = parent.getDescriptor().createAuthenticationProvider();
this.timestamp = timestamp;
this.update = update;
this.listener = listener;
this.locations = parent.getLocations();
}
public List invoke(File ws, VirtualChannel channel) throws IOException {
final SVNClientManager manager = createSvnClientManager(authProvider);
try {
final SVNUpdateClient svnuc = manager.getUpdateClient();
final List externals = new ArrayList(); // store discovered externals to here
//Patched -using HEAD instead of timestamp version
final SVNRevision revision = SVNRevision.HEAD;
// final SVNRevision revision = SVNRevision.create(timestamp);
if(update) {
for (final ModuleLocation l : locations) {
try {
listener.getLogger().println("Updating "+ l.remote);
svnuc.setEventHandler(new SubversionUpdateEventHandler(listener.getLogger(), externals, l.local));
svnuc.doUpdate(new File(ws, l.local).getCanonicalFile(), revision, true);
} catch (final SVNException e) {
e.printStackTrace(listener.error("Failed to update "+l.remote));
// trouble-shooting probe for #591
if(e.getErrorMessage().getErrorCode()== SVNErrorCode.WC_NOT_LOCKED) {
listener.getLogger().println("Polled jobs are "+ SCMTrigger.DESCRIPTOR.getItemsBeingPolled());
}
return null;
}
}
} else {
Util.deleteContentsRecursive(ws);
PipedOutputStream pos = new PipedOutputStream();
PipedInputStream pis = new PipedInputStream(pos);
StreamCopyThread sct = new StreamCopyThread("svn log copier", pis, listener.getLogger());
sct.start();
for (final ModuleLocation l : locations) {
try {
final SVNURL url = SVNURL.parseURIEncoded(l.remote);
listener.getLogger().println("Checking out "+url);
svnuc.setEventHandler(new SubversionUpdateEventHandler(new PrintStream(pos), externals, l.local));
svnuc.doCheckout(url, new File(ws, l.local).getCanonicalFile(), SVNRevision.HEAD, revision, true);
} catch (final SVNException e) {
e.printStackTrace(listener.error("Failed to check out "+l.remote));
return null;
}
}
pos.close();
try {
sct.join(); // wait for all data to be piped.
} catch (InterruptedException e) {
}
}
return externals;
} finally {
manager.dispose();
}
}
private static final long serialVersionUID = 1L;
}
/**
* Creates {@link SVNClientManager}.
*
*
* This method must be executed on the slave where svn operations are performed.
*
* @param authProvider
* The value obtained from {@link DescriptorImpl#createAuthenticationProvider()}.
* If the operation runs on slaves,
* (and properly remoted, if the svn operations run on slaves.)
*/
/*package*/ static SVNClientManager createSvnClientManager(ISVNAuthenticationProvider authProvider) {
ISVNAuthenticationManager sam = SVNWCUtil.createDefaultAuthenticationManager();
sam.setAuthenticationProvider(authProvider);
return SVNClientManager.newInstance(SVNWCUtil.createDefaultOptions(true),sam);
}
public static final class SvnInfo implements Serializable, Comparable {
/**
* Decoded repository URL.
*/
public final String url;
public final long revision;
public SvnInfo(String url, long revision) {
this.url = url;
this.revision = revision;
}
public SvnInfo(SVNInfo info) {
this( info.getURL().toDecodedString(), info.getCommittedRevision().getNumber() );
}
public SVNURL getSVNURL() throws SVNException {
return SVNURL.parseURIDecoded(url);
}
public int compareTo(SvnInfo that) {
int r = this.url.compareTo(that.url);
if(r!=0) return r;
if(this.revisionthat.revision) return +1;
return 0;
}
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
SvnInfo svnInfo = (SvnInfo) o;
if (revision != svnInfo.revision) return false;
return url.equals(svnInfo.url);
}
public int hashCode() {
int result;
result = url.hashCode();
result = 31 * result + (int) (revision ^ (revision >>> 32));
return result;
}
public String toString() {
return String.format("%s (rev.%s)",url,revision);
}
private static final long serialVersionUID = 1L;
}
/**
* Gets the SVN metadata for the given local workspace.
*
* @param workspace
* The target to run "svn info".
*/
private static SVNInfo parseSvnInfo(File workspace, ISVNAuthenticationProvider authProvider) throws SVNException {
final SVNClientManager manager = createSvnClientManager(authProvider);
try {
final SVNWCClient svnWc = manager.getWCClient();
return svnWc.doInfo(workspace,SVNRevision.WORKING);
} finally {
manager.dispose();
}
}
/**
* Gets the SVN metadata for the remote repository.
*
* @param remoteUrl
* The target to run "svn info".
*/
private static SVNInfo parseSvnInfo(SVNURL remoteUrl, ISVNAuthenticationProvider authProvider) throws SVNException {
final SVNClientManager manager = createSvnClientManager(authProvider);
try {
final SVNWCClient svnWc = manager.getWCClient();
return svnWc.doInfo(remoteUrl, SVNRevision.HEAD, SVNRevision.HEAD);
} finally {
manager.dispose();
}
}
/**
* Checks .svn files in the workspace and finds out revisions of the modules
* that the workspace has.
*
* @return
* null if the parsing somehow fails. Otherwise a map from the repository URL to revisions.
*/
private static class BuildRevisionMapTask implements FileCallable