# Part of the A-A-P recipe executive: Generic Version Control # Copyright (C) 2002-2003 Stichting NLnet Labs # Permission to copy and use this file is specified in the file COPYING. # If this file is missing you can find it here: http://www.a-a-p.org/COPYING # # Functions to get files out of a version control system and put them back. # Most of the work is directed to one of the modules for a specific version # control system. # Uploading/downloading is done by functions in Remote.py. # import os import os.path import string from Process import recipe_error from VersContCvs import * from Work import getwork from Node import Node from Remote import download_file, upload_file, remote_remove def separate_scheme(url): """Isolate the scheme of the URL.""" i = string.find(url, "://") if i < 0: # Catch "file:~user". if len(url) > 5 and url[:5] == "file:": return "file", url[5:] raise UserError, _('No :// found in "%s"') % url scheme = string.lower(url[:i]) for c in scheme: if not c in string.lowercase: raise UserError, _('Illegal character before colon in "%s"') % url return scheme, url[i + 3:] def repl_file_name(attr, name): """Replace all "%file%" in "attr" with "name".""" while 1: i = string.find(attr, "%file%") if i >= 0: attr = attr[:i] + name + attr[i + 6:] continue i = string.find(attr, "%basename%") if i >= 0: attr = attr[:i] + os.path.basename(name) + attr[i + 10:] continue break return attr def verscont_command(recdict, dict, nodelist, use_cache, action): """ Try performing "action" on nodes "nodelist" from the semi-URL "dict["name"]". May Use a cached file when "use_cache" is non-zero. Returns list of nodes that failed. """ scheme, name = separate_scheme(dict["name"]) # Handle a scheme for which there is a scheme_command() function. fun = recdict["_no"].get(scheme + "_command") if fun: return apply(fun, (recdict, name, dict, nodelist, action)) # Handle the "cvs://" scheme. if scheme == "cvs": return cvs_command(recdict, name, dict, nodelist, action) # Handle uploading. if action in ["commit", "checkin", "publish", "add"]: return upload_file(recdict, dict, nodelist) failed = [] for node in nodelist: # Assume it's a normal URL, try downloading/uploading/removing the # file. name = dict["name"] dict["name"] = repl_file_name(name, node.name) if action in ["fetch", "checkout"]: ok = download_file(recdict, dict, node, use_cache) elif action == "remove": ok = remote_remove(recdict, dict, node) else: ok = 0 if not ok: failed.append(node) # restore the name, %file% might have to be replaced again dict["name"] = name return failed def handle_nodelist(rpstack, recdict, nodelist, use_cache, action, attrnames): """Common code for fetching, committing, etc. "action" is the name of the command: "fetch", "commit", etc. Perform this action on each node in "nodelist", unless "--nobuild" or "--touch" is used. "attrnames" is a list of attribute names that can be used for this action. Return list of failed nodes. """ faillist = [] # Make a copy of the list of nodes. Items are removed when done. todolist = nodelist[:] try: groupcount = int(get_var_val(0, recdict, "_no", "GROUPCOUNT")) except StandardError, e: recipe_error(rpstack, _('Invalid $GROUPCOUNT value: %s: %s') % (str(get_var_val(0, recdict, "_no", "GROUPCOUNT")), str(e))) while todolist: # Find a bunch of nodes with identical attributes and handle them all # at once. attr = '' thislist = [] this_use_cache = None for node in todolist[:]: # Use the first attribute in "attrnames" that exists and isn't # empty. nx_attr = '' for n in attrnames: if node.attributes.has_key(n): nx_attr = node.attributes[n] if nx_attr: attrname = n break # Using cache for this node? uc = use_cache if (node.attributes.get("usecache") or node.attributes.get("constant")): uc = 1 if node.attributes.get("nocache"): uc = 0 if not nx_attr: recipe_error(rpstack, _("Missing %s attribute for %s") % (attrnames[0], node.short_name())) todolist.remove(node) faillist.append(node) elif ((not attr or nx_attr == attr) and (this_use_cache is None or this_use_cache == uc)): thislist.append(node) todolist.remove(node) attr = nx_attr this_use_cache = uc # Limit to 20 to avoid very long command lines. if len(thislist) >= groupcount: break # Do the list of nodes with identical attributes "attr". There is at # least one. from Dictlist import str2dictlist alist = str2dictlist(rpstack, attr) if not alist: recipe_error(rpstack, _("%s attribute for %s is empty") % (attrname, thislist[0].short_name())) # Perform the action, unless started with --nobuild or --touch. if skip_commands(): if not Global.cmd_args.options.get("touch"): msg_info(recdict, _('skip %s for %s') % (action, str(map(lambda x: x.short_name(), thislist)))) else: # Loop over the list of locations. Quit as soon as one worked, # except for "publish", all locations are used then. for loc in alist: # Try handling thislist with this location. For publish do # this for all locations, otherwise stop when all items have # been done. res = verscont_command(recdict, loc, thislist, this_use_cache, action) if action == "publish": for r in res: if not r in faillist: faillist.append(r) else: thislist = res if not thislist: break if action != "publish" and thislist: # this bunch failed. Still continue doing the remaining ones. faillist.extend(thislist) return faillist def fetch_nodelist(rpstack, recdict, nodelist, use_cache): """Fetch nodes in "nodelist" according to its "fetch" attribute. When there is no "fetch" attribute use "commit". Only use cached files when "use_cache" is non-zero. Return list of failed nodes.""" return handle_nodelist(rpstack, recdict, nodelist, use_cache, "fetch", [ "fetch", "commit" ]) def verscont_nodelist(rpstack, recdict, nodelist, action): """Checkout nodes in "nodelist" according to their "commit" attribute. Return list of failed nodes.""" return handle_nodelist(rpstack, recdict, nodelist, 0, action, [ "commit" ]) def publish_nodelist(rpstack, recdict, nodelist, msg): """Publish nodes in "nodelist" according to their "publish" attribute. When there is no "publish" attribute use "commit". When "msg" is non-zero give a message for up-to-date nodes. Return None for nothing done, list of failed nodes otherwise. """ # Publishing is only done when the signature is outdated. # Need to pull a few tricks to be able to call check_need_update(). from Sign import sign_updated, buildcheck_updated from DoBuild import Update, check_need_update todolist = [] buildchecklist = [] targetlist = [] failed = [] for node in nodelist: # Skip nodes that don't exist. node_name = node.get_name() if not os.path.exists(node_name): failed.append(node) msg_error(recdict, _('Published file does not exist: "%s"') % node_name) continue if node.attributes.has_key("publish"): s = node.attributes["publish"] elif node.attributes.has_key("commit"): s = node.attributes["commit"] else: s = '' # Handle each publish/commit item separately, so that when adding a new # destination we don't upload to the old destinations as well. from Dictlist import str2dictlist, dictlist2str alist = str2dictlist(rpstack, s) if not alist: recipe_error(rpstack, _('publish attribute for "%s" is empty') % node.short_name()) for attr in alist: # Use a fake virtual target to attach the signature to. # Use the "remember" attribute, only publish when changed. # Use the sign file in the directory related to the published file. attr_name = attr["name"] attr_str = dictlist2str([ attr ]) # Change "scp://" and "rscync://" to "scheme://", so that changing # the method for uploading doesn't cause the signature to be # outdated. attr_name = re.sub("^[a-z]*://", "scheme://", attr_name) buildcheck_str = re.sub("\\b[a-z]*://", "scheme://", attr_str) target = Node(":publish:" + node_name + ">" + attr_name) target.attributes["virtual"] = 1 target.attributes["remember"] = 1 target.attributes["signfile"] = node.get_sign_fname() source = node.copy() source.attributes["_node"] = source source.attributes["name"] = node_name source.attributes["publish"] = attr_str # Need to publish the file when it's signature has changed or the # relevant attributes have changed (use buildcheck for this). update = Update() check_need_update(recdict, update, source.attributes, target, rootname = ":publish:" + node_name + ">") update.set_buildcheck(recdict, buildcheck_str, target, 0) # Find out if there is a reason to publish this source node. reason = update.upd_reason(0, target) if reason or msg: disp_name = node.short_name() + "[" + attr_str + "]" if reason: if Global.cmd_args.options.get("touch"): msg_info(recdict, _('Touching "%s": %s') % (disp_name, reason)) else: msg_depend(recdict, _('Publishing "%s": %s') % (disp_name, reason)) todolist.append(source) buildchecklist.append(update.buildcheck) targetlist.append(target) else: if msg: msg_depend(recdict, _('item is up-to-date: "%s"') % disp_name) # For the "--contents" option we skip items for which the # "publish" attribute changed but not the contents. Need to # update the signature anyway, otherwise it will still be # published next time. if (Global.cmd_args.options.get("contents") and not Global.cmd_args.options.get("nobuild")): sign_updated(recdict, source, source.attributes, target) if update.buildcheck: buildcheck_updated(recdict, target, update.buildcheck) if not todolist: if failed: return failed # all outdated files do not exist return None # all files exist and targets are up-to-date failed.extend(handle_nodelist(rpstack, recdict, todolist, 0, "publish", [ "publish", "commit" ])) # Update signatures, unless --nobuild was used withouth --touch. if (not Global.cmd_args.options.get("nobuild") or Global.cmd_args.options.get("touch")): i = 0 while i < len(todolist): # Remember the source signature and the buildcheck signature. # But only for nodes that didn't fail. node = todolist[i] if not node in failed: target = targetlist[i] buildcheck = buildchecklist[i] sign_updated(recdict, node, node.attributes, target) if buildcheck: buildcheck_updated(recdict, target, buildcheck) i = i + 1 return failed def verscont_remove_add(rpstack, recdict, dir, recursive, action): """When "action" is "remove: Remove all files in directory "dir" of VCS that don't belong there. When "action" is "add:" Add all files in the recipe in directory "dir" to the VCS that are missing. "dir" is a dictionary for the directory and its attributes. Enter directories recursively when "recursive" is non-zero. """ if not dir.has_key("commit"): recipe_error(rpstack, _("no commit attribute for %s") % dir["name"]) from Dictlist import str2dictlist commit_list = str2dictlist(rpstack, dir["commit"]) if not commit_list: recipe_error(rpstack, _("commit attribute for %s is empty") % dir["name"]) dirname = os.path.abspath(dir["name"]) alist = None for commit_item in commit_list: scheme, name = separate_scheme(commit_item["name"]) # Handle a scheme for which there is a scheme_list() function. fun = recdict["_no"].get(scheme + "_list") if fun: alist = apply(fun, (recdict, name, commit_item, dirname, recursive)) break # Handle the "cvs://" scheme. if scheme == "cvs": alist = cvs_list(recdict, name, commit_item, dirname, recursive) break if alist is None: recipe_error(rpstack, _("No working item in commit attribute for %s") % dir["name"]) work = getwork(recdict) ok = 1 if action == "remove": # Remove: Loop over all items found in the VCS. for item in alist: node = work.find_node(item) if not node or not node.attributes.has_key("commit"): if not node: node = Node(item) if skip_commands(): if not Global.cmd_args.options.get("touch"): msg_info(recdict, _('Remove "%s"') % node.short_name()) elif verscont_command(recdict, commit_item, [ node ], 0, "remove"): ok = 0 else: # Add: Loop over all nodes and check if they are in the VCS. # Only do this for a node that: # - has the "commit" attribute # - has a full name that is longer than the directory name # - working recursive and the node is below the directory # - not working recursive and the node is in the directory # - the node does not appear in the list from the VCS dirname_len = len(dirname) for node in work.nodes.values(): if (node.attributes.has_key("commit") and len(node.absname) > dirname_len and ((recursive and node.absname[:dirname_len] == dirname and node.absname[dirname_len] == '/') or (not recursive and os.path.dirname(node.absname) == dirname)) and not node.absname in alist): if skip_commands(): if not Global.cmd_args.options.get("touch"): msg_info(recdict, _('Add "%s"') % node.short_name()) elif verscont_command(recdict, commit_item, [ node ], 0, "add"): ok = 0 return ok # vim: set sw=4 et sts=4 tw=79 fo+=l: