# Part of the A-A-P recipe executive: copy and move files (remotely) # Copyright (C) 2002-2003 Stichting NLnet Labs # Permission to copy and use this file is specified in the file COPYING. # If this file is missing you can find it here: http://www.a-a-p.org/COPYING # # These are functions to copy and move files. Not only on the local system, # also to remote systems, using http://, ftp://, etc. # import os import os.path import string import re from Dictlist import str2dictlist from Error import * from Process import recipe_error, option_error from Util import * from Work import getrpstack from Message import * def copy_move(line_nr, recdict, arg, copy): """Implementation of ":copy -x from to" and ":move -x from to". When "copy" is non-zero copying is done, otherwise moving. "arg" is the whole argument. "line_nr" is used for error messages.""" # Skip when not actually building. cmdname = (copy and ':copy') or ':move' if skip_commands(): msg_skip(line_nr, recdict, cmdname + ' ' + arg) return rpstack = getrpstack(recdict, line_nr) # Get the arguments and check the options. # TODO: check if all options are handled properly from Commands import get_args opt = {"f": "force", "force": "force", "i": "interactive", "interactive": "interactive", "e": "exist", "exist": "exist", "exists": "exist", "m": "mkdir", "mkdir": "mkdir", "c": "continue", "continue": "continue", "q": "quiet", "quiet": "quiet"} if copy: opt.update({ "u": "unlink", "unlink": "unlink", "p": "preserve", "preserve": "preserve", "r": "recursive", "recursive": "recursive", "k": "keepdir", "keepdir": "keepdir"}) optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg, opt) if attrdict: option_error(rpstack, attrdict, cmdname) if len(argdictlist) < 2: recipe_error(rpstack, _("%s command requires at least two arguments") % cmdname) remote_copy_move(rpstack, recdict, copy, argdictlist[:-1], argdictlist[-1], optiondict, 1, errmsg = 1) # Schemes handled by scheme_copy(). When changing this also need to change # code and comments below! scheme_copy_names = ["rcp", "scp", "rsync"] def scheme_copy(rpstack, recdict, scheme, fromfile, tmach, destpath, fcount): """ Use rcp, scp or rsync, specified with "scheme" to copy "fromfile" (with "fcount" file names in quotes) to tmach:destpath. Return non-zero for success. """ # Install rcp/scp/rsync when needed. from DoInstall import assert_pkg assert_pkg(rpstack, recdict, scheme) from Remote import get_progname_rsync, get_progname_rcp, get_progname_scp if scheme == "rcp": cmd = "%s %s '%s:%s'" % (get_progname_rcp(recdict), fromfile, tmach, destpath) elif scheme == "scp": cmd = "%s %s '%s:%s'" % (get_progname_scp(recdict), fromfile, tmach, destpath) else: # A bug in rsync: it can't handle "file(with)parenthesis" in the # destination, but escaping them in the source is not allowed! d = re.sub("([()])", r"\\\1", destpath) cmd = "%s %s '%s:%s'" % (get_progname_rsync(recdict), fromfile, tmach, d) if os.name == "posix": # On Unix we can use "tee". ok, text = redir_system_int(recdict, cmd) else: # Scp may ask for a password. When we can't use "tee" we need to # see the message. Can't catch the text output then... ok = logged_system(recdict, "{interactive} " + cmd) == 0 text = '' if text: msg_log(recdict, text) # Check for an error connecting, scp doesn't exit with an error value # then. Usual messages are: # "Secure connection to vim.sf.net refused." # "lost connection" # "Read-only file system" i = string.find(text, "connection to") if i > 0: e = string.find(text, "\n", i) if e > 0: i = string.find(text, "refused", i, e) else: i = -1 if i > 0 or string.find(text, "lost connection") >= 0: ok = 0 elif string.find(text, "Permission denied") >= 0: # Happens when a file exists and permissions don't allow # overwriting: scp: vim/new/con_cvs.php: Permission denied # TODO: check the file name and only mark this one as failed. ok = 0 elif string.find(text, "Read-only file system") >= 0: # Happens when the file system was mounted read-only. # TODO: check the file name and only mark this one as failed. ok = 0 elif string.find(text, "Not a directory") >= 0: # Happens when copying multiple files into a file. ok = 0 elif string.find(text, "No such file or directory") >= 0: ok = 0 msg_info(recdict, _('looks like the directory does not exist, creating it')) # TODO: This is just guessing; how do we know if the destination # is a directory or a file? if fcount > 1: dirname = destpath else: dirname = os.path.dirname(destpath) mkdircmd = ('ssh %s mkdir -p %s' % (tmach, dirname)) if logged_system(recdict, mkdircmd) == 0: # Try copying again. ok = (logged_system(recdict, cmd) == 0) return ok # Cached ftp connections. ftp_conn = {} did_set_exitfunc = 0 old_exitfunc = None def ftpCloseAll(): """ Close all cached ftp connections. Invoked on exit. """ import ftplib if old_exitfunc: old_exitfunc() # Invoke previous exit function. global ftp_conn for ftp in ftp_conn.values(): try: ftp.quit() except ftplib.all_errors, e: msg_log({}, _("could not quit an ftp connection: %s") % e) ftp_conn = {} # Cached passwords for ftp connections. ftp_pass = {} def ftpConnect(mach): """ Connect to an ftp server "mach". If "mach" is in the form "user:passwd@ftp.com" use that. Otherwise consult the netrc file or ask the user. Returns a tuple. The first item is an ftp object. If the connection was successful the second item is an empty string. If the connection failed the second item is an error message. """ # Use a cached connection if there is one. if mach in ftp_conn.keys(): return ftp_conn[mach], '' passwd = '' acct = '' key = None msg = '' # For "user:passwd@ftp.com" split at the @ i = string.find(mach, '@') if i > 0: user = mach[:i] machname = mach[i+1:] i = string.find(user, ':') if i > 0: passwd = user[i+1:] user = user[:i] else: key = user + "@" + machname passwd = ftp_pass.get(key) if not passwd: prompt = (_('Enter password for user %s at %s: ') % (user, machname)) try: import getpass passwd = getpass.getpass(prompt) except: # TODO: should display stars for typed chars passwd = raw_input(prompt) else: user = '' machname = mach import aapnetrc # obtain the login name and password from the netrc file try: n = aapnetrc.netrc() res = n.authenticators(machname) if res is None: user = '' else: user, acct, passwd = res except (IOError, aapnetrc.NetrcParseError), e: pass # Try to open the connection to the ftp server. import ftplib try: ftp = ftplib.FTP() i = string.find(machname, ':') if i > 0: # Port number specified. ftp.connect(machname[0:i], machname[i+1:]) else: # Use default port number. ftp.connect(machname) if user != '': if passwd != '': if acct != '': ftp.login(user, passwd, acct) else: ftp.login(user, passwd) else: ftp.login(user) else: ftp.login() except ftplib.all_errors, e: msg = (_('Cannot open connection to "%s": %s') % (mach, str(e))) else: if key and passwd: # Remember the typed password, so that the user doesn't have to # type it again. ftp_pass[key] = passwd # Cache the connection for another time. ftp_conn[mach] = ftp # Setup the exit function to close the connection. global did_set_exitfunc global old_exitfunc if not did_set_exitfunc: did_set_exitfunc = 1 old_exitfunc = getattr(sys, 'exitfunc', None) sys.exitfunc = ftpCloseAll return ftp, msg def ftp_may_mkdir(recdict, error, ftp, destpath): """If "error" indicates we failed to upload through ftp because the directory doesn't exist try creating that directory. This works recursively. Return non-zero if the directory was created.""" if string.find(str(error), "No such file or dir") >= 0: adir = os.path.dirname(destpath) if len(adir) < len(destpath): import ftplib msg_info(recdict, _('Directory "%s" does not appear to exist, creating it...') % adir) try: ftp.mkd(adir) except ftplib.all_errors, e: # Maybe the upper directory doesn't exist, try creating it. if ftp_may_mkdir(recdict, e, ftp, adir): try: ftp.mkd(adir) except ftplib.all_errors, e: msg_info(recdict, _('Could not create directory "%s"') % adir) return 0 else: return 0 return 1 return 0 def remote_copy_move(rpstack, recdict, copy, from_items, to_item, optiondict, argexpand, errmsg = 0): """Copy or move command. Copy when "copy" is non-zero. "from_items" is a dictlist of the files to copy or move from. "to_item" is the dictionary of the destination. "optiondict" is a dictionary with options: recursive: recursive copy. exist: skip if destination exists interactive: ask about overwriting. preserve: keep mode bits and timestamp quiet: don't report copied/moved files continue: ignore wildcards without matches mkdir: create directory when needed "argexpand" is non-zero when wildcards are to be expanded (called from ":copy" or ":move"). When "errmsg" is non-zero, call recipe_error() for an error. Returns a list of filenames that failed.""" from Remote import url_split3 import glob failed = [] # -l: change symlinks to real files #if optiondict.get("unlink"): # keepsymlinks = 0 #else: # keepsymlinks = 1 # Expand and check the "from" arguments. fromlist = [] for a in from_items: fname = a["name"] fscheme, fmach, fpath = url_split3(fname) if fscheme == '': # It's a local file, may expand ~user and wildcards. f = os.path.expanduser(fname) if argexpand: fl = glob.glob(f) if len(fl) == 0: if optiondict.get("continue") and has_wildcard(f): continue recipe_error(rpstack, _('No match for "%s"') % fname) else: fl = [ f ] # For copy without {r} sources can't be a directory. if copy and not optiondict.get("recursive"): for l in fl: if os.path.isdir(l): recipe_error(rpstack, _('Copying a directory requires {r} flag: %s') % l) fromlist.extend(fl) else: # It's a URL, append without expanding. if not copy: recipe_error(rpstack, _('Cannot move from a URL yet: "%s"') % fname) if argexpand: # Do change [*] to *, we might support expanding later. f = expand_unescape(fname) else: f = fname fromlist.append(f) # Expand and check the "to" argument. tname = to_item["name"] tscheme, tmach, tpath = url_split3(tname) if tscheme == '': # For a local file expand "~/" and "~user/". Don't use "tname", it may # start with "file:". tpath = os.path.expanduser(tpath) if argexpand: if tscheme == '': # For a local destination file expand wildcards. If expanding # results in no matches use the name without expansion. l = glob.glob(tpath) if len(l) > 1: recipe_error(rpstack, _('More than one match for "%s"') % tname) if len(l) == 1: tpath = l[0] else: # Can't expand wildcards in a URL yet, at least remove escaped # wildcards. tpath = expand_unescape(tpath) if tscheme == '' and optiondict.get("mkdir"): # Create local target directory when it doesn't exist. if len(fromlist) > 1: dirpath = tpath; else: dirpath = os.path.dirname(tpath) may_create_dir(recdict, rpstack, dirpath, optiondict.get("quiet")) # If there is more than one source, target must be a directory. if tscheme == '' and len(fromlist) > 1 and not os.path.isdir(tpath): recipe_error(rpstack, _('Destination must be a directory: "%s"') % tname) msg = '' if tscheme == "ftp": # # Prepare for uploading through ftp. # ftp, msg = ftpConnect(tmach) elif tscheme in scheme_copy_names: # # TODO: Prepare for uploading through rcp/scp/rsync. # pass elif tscheme != '': msg = _('Can only upload to rcp://, scp://, rsync:// and ftp://') # If copy/move isn't possible either give an error message or return the # whole list of sources. if msg: if argexpand: msg_error(recdict, msg) return map(lambda x : x["name"], from_items) recipe_error(rpstack, msg) # files to be copied with scp or rsync later. scheme_files = {} scheme_filelist = {} for scheme in scheme_copy_names: scheme_files[scheme] = '' scheme_filelist[scheme] = [] # # Loop over all "from" files. # for fname in fromlist: fscheme, fmach, fpath = url_split3(fname) # If the destination is a directory, append the source file name to the # destination directory. # "dest" includes the scheme (may be "file:" when "tscheme" is empty). if ((tscheme != '' and len(fromlist) > 1) or (tscheme == '' and os.path.isdir(tpath))): if optiondict.get("keepdir"): dest = os.path.join(tname, fpath) destpath = os.path.join(tpath, fpath) if tscheme == '' and os.path.dirname(fpath): # Create the directory if it doesn't exist. dirpath = os.path.dirname(destpath) may_create_dir(recdict, rpstack, dirpath, optiondict.get("quiet")) else: dest = os.path.join(tname, os.path.basename(fname)) destpath = os.path.join(tpath, os.path.basename(fname)) else: dest = tname destpath = tpath # If destination is a local file and exists: # - When {exist} option used, silently ignore. # - When {interactive} option used, ask for overwriting. Use a special # string to allow translating the response characters. if tscheme == '' and os.path.exists(destpath): if optiondict.get("exist"): msg = _('file already exists: "%s"') % dest msg_note(recdict, msg) failed.append(fname) continue if optiondict.get("interactive"): reply = raw_input(_('"%s" exists, overwrite? (y/n) ') % dest) if (len(reply) == 0 or not reply[0] in _("yY up to four chars that mean yes")[:4]): if copy: msg_note(recdict, _("file not copied")) else: msg_note(recdict, _("file not moved")) failed.append(fname) continue if fscheme == '' and tscheme == '': # # local file copy or move # if not copy: try: os.rename(fpath, destpath) done = 1 except: done = 0 # renaming failed, try copying if copy or not done: try: import shutil # TODO: handle symlinks and "keepsymlinks". if os.path.isdir(fpath): # Cannot use shutil.copytree here, it fails when the # destination already exists. if not os.path.exists(destpath): os.mkdir(destpath) # Call ourselves recursively from Commands import dir_contents flist = map(lambda x: {"name": x}, dir_contents(fpath)) if flist: # Remove the "keepdir" attribute, the directory # name is already in "destpath" now. if optiondict.has_key("keepdir"): opt = optiondict.copy() del opt["keepdir"] else: opt = optiondict f = remote_copy_move(rpstack, recdict, copy, flist, {"name": destpath}, opt, 0, errmsg) if f: failed.extend(f) elif not copy or optiondict.get("preserve"): shutil.copy2(fpath, destpath) else: shutil.copy(fpath, destpath) except (IOError, OSError), e: msg = (_('Cannot copy "%s" to "%s": %s') % (fname, dest, str(e))) if not argexpand and not errmsg: msg_note(recdict, msg) failed.append(fname) else: recipe_error(rpstack, msg) continue if not copy: if os.path.isdir(fpath): deltree(fpath) else: os.remove(fpath) if not optiondict.get("quiet"): f = shorten_name(fpath) if copy: msg_info(recdict, _('Copied "%s" to "%s"') % (f, dest)) else: msg_info(recdict, _('Moved "%s" to "%s"') % (f, dest)) else: if fscheme != '': # download to local file from Remote import url_download try: if tscheme != '': tmpfile, rtime = url_download(recdict, fname, '') else: tmpfile, rtime = url_download(recdict, fname, destpath) except IOError, e: msg = (_('Cannot download "%s" to "%s": %s') % (fname, dest, str(e))) if not argexpand and not errmsg: msg_note(recdict, msg) failed.append(fname) else: recipe_error(rpstack, msg) continue if tscheme != '': if fscheme != '': # use temporary file fromfile = tmpfile else: fromfile = fpath postponed = 0 if tscheme == 'ftp': # No system command, give a message about what we're doing, # it may take a while. if fscheme != '': msg_info(recdict, _('Uploading to "%s"' % dest)) else: if not copy: msg_info(recdict, _('Moving "%s" to "%s"' % (fname, dest))) else: msg_info(recdict, _('Uploading "%s" to "%s"' % (fname, dest))) # FTP must use '/' path separator characters, # os.path.join() may have used something else. if os.sep != '/': destpath = string.replace(destpath, os.sep, '/') def store_file(ftp, fromfile, destpath): import ftplib # avoid warning message error = None try: f = open(fromfile, "rb") except IOError, e: return e try: ftp.storbinary("STOR " + destpath, f, 8192) except ftplib.all_errors, e: error = e f.close() return error error = store_file(ftp, fromfile, destpath) # If it looks like the directory doesn't exist, try # creating it. if error and ftp_may_mkdir(recdict, error, ftp, destpath): msg_info(recdict, _('Directory created, attempt uploading again...')) error = store_file(ftp, fromfile, destpath) if error: msg = (_('Cannot upload "%s" to "%s": %s') % (fname, dest, str(error))) if not argexpand and not errmsg: msg_note(recdict, msg) failed.append(fname) else: recipe_error(rpstack, msg) if fscheme != '': os.remove(tmpfile) # delete temporary file continue elif tscheme in scheme_copy_names: if fscheme == '' and len(fromlist) > 1: # do all local files at once below scheme_files[tscheme] = (scheme_files[tscheme] + "'" + fromfile + "' ") scheme_filelist[tscheme].append(fromfile) postponed = 1 elif not scheme_copy(rpstack, recdict, tscheme, "'" + fromfile + "'", tmach, destpath, 1): msg = _('Cannot upload "%s" to "%s"') % (fname, dest) if not argexpand and not errmsg: msg_note(recdict, msg) failed.append(fname) else: recipe_error(rpstack, msg) if fscheme != '': os.remove(tmpfile) # delete temporary file continue if fscheme != '': os.remove(tmpfile) # delete temporary file msg_info(recdict, _('Uploaded to "%s"' % dest)) else: if not copy: os.remove(fname) msg_info(recdict, _('Moved "%s" to "%s"' % (shorten_name(fname), dest))) elif not postponed: msg_info(recdict, _('Uploaded "%s" to "%s"' % (shorten_name(fname), dest))) # End of loop over all "from" files. if fscheme != '': from Remote import url_cleanup url_cleanup(fscheme) # Copy collected files with rcp, scp and rsync. for scheme in scheme_copy_names: files = scheme_files[scheme] if files: filelist = scheme_filelist[scheme] # If there is only one file to copy, add its basename to the # destination path to avoid creating a file by the name of the # directory of that file. if len(filelist) == 1: tpath = os.path.join(tpath, os.path.basename(filelist[0])) if scheme_copy(rpstack, recdict, scheme, files, tmach, tpath, len(filelist)): dl = shorten_dictlist(str2dictlist(rpstack, files)) flist = map(lambda x: x["name"], dl) msg_info(recdict, _('Uploaded "%s" to "%s"' % (flist, tname))) else: msg = _('Cannot upload "%s" to "%s"') % (files, tname) if not argexpand and not errmsg: msg_note(recdict, msg) failed.extend(filelist) else: recipe_error(rpstack, msg) return failed def may_create_dir(recdict, rpstack, dir, quiet): """ Create directory "dir" if it does not exist. """ if not os.path.exists(dir): try: os.makedirs(dir) if not quiet: msg_info(recdict, _('Created directory "%s"') % dir) except StandardError, e: recipe_error(rpstack, _('Destination directory "%s" cannot be created: %s') % (dir, str(e))) # vim: set sw=4 et sts=4 tw=79 fo+=l: