# Part of the A-A-P recipe executive: Generic Version Control
# Copyright (C) 2002-2003 Stichting NLnet Labs
# Permission to copy and use this file is specified in the file COPYING.
# If this file is missing you can find it here: http://www.a-a-p.org/COPYING
#
# Functions to get files out of a version control system and put them back.
# Most of the work is directed to one of the modules for a specific version
# control system.
# Uploading/downloading is done by functions in Remote.py.
#
import os
import os.path
import string
from Process import recipe_error
from VersContCvs import *
from Work import getwork
from Node import Node
from Remote import download_file, upload_file, remote_remove
def separate_scheme(url):
"""Isolate the scheme of the URL."""
i = string.find(url, "://")
if i < 0:
# Catch "file:~user".
if len(url) > 5 and url[:5] == "file:":
return "file", url[5:]
raise UserError, _('No :// found in "%s"') % url
scheme = string.lower(url[:i])
for c in scheme:
if not c in string.lowercase:
raise UserError, _('Illegal character before colon in "%s"') % url
return scheme, url[i + 3:]
def repl_file_name(attr, name):
"""Replace all "%file%" in "attr" with "name"."""
while 1:
i = string.find(attr, "%file%")
if i >= 0:
attr = attr[:i] + name + attr[i + 6:]
continue
i = string.find(attr, "%basename%")
if i >= 0:
attr = attr[:i] + os.path.basename(name) + attr[i + 10:]
continue
break
return attr
def verscont_command(recdict, dict, nodelist, use_cache, action):
"""
Try performing "action" on nodes "nodelist" from the semi-URL
"dict["name"]".
May Use a cached file when "use_cache" is non-zero.
Returns list of nodes that failed.
"""
scheme, name = separate_scheme(dict["name"])
# Handle a scheme for which there is a scheme_command() function.
fun = recdict["_no"].get(scheme + "_command")
if fun:
return apply(fun, (recdict, name, dict, nodelist, action))
# Handle the "cvs://" scheme.
if scheme == "cvs":
return cvs_command(recdict, name, dict, nodelist, action)
# Handle uploading.
if action in ["commit", "checkin", "publish", "add"]:
return upload_file(recdict, dict, nodelist)
failed = []
for node in nodelist:
# Assume it's a normal URL, try downloading/uploading/removing the
# file.
name = dict["name"]
dict["name"] = repl_file_name(name, node.name)
if action in ["fetch", "checkout"]:
ok = download_file(recdict, dict, node, use_cache)
elif action == "remove":
ok = remote_remove(recdict, dict, node)
else:
ok = 0
if not ok:
failed.append(node)
# restore the name, %file% might have to be replaced again
dict["name"] = name
return failed
def handle_nodelist(rpstack, recdict, nodelist, use_cache, action, attrnames):
"""Common code for fetching, committing, etc.
"action" is the name of the command: "fetch", "commit", etc.
Perform this action on each node in "nodelist", unless "--nobuild" or
"--touch" is used.
"attrnames" is a list of attribute names that can be used for this
action.
Return list of failed nodes.
"""
faillist = []
# Make a copy of the list of nodes. Items are removed when done.
todolist = nodelist[:]
try:
groupcount = int(get_var_val(0, recdict, "_no", "GROUPCOUNT"))
except StandardError, e:
recipe_error(rpstack, _('Invalid $GROUPCOUNT value: %s: %s')
% (str(get_var_val(0, recdict, "_no", "GROUPCOUNT")), str(e)))
while todolist:
# Find a bunch of nodes with identical attributes and handle them all
# at once.
attr = ''
thislist = []
this_use_cache = None
for node in todolist[:]:
# Use the first attribute in "attrnames" that exists and isn't
# empty.
nx_attr = ''
for n in attrnames:
if node.attributes.has_key(n):
nx_attr = node.attributes[n]
if nx_attr:
attrname = n
break
# Using cache for this node?
uc = use_cache
if (node.attributes.get("usecache")
or node.attributes.get("constant")):
uc = 1
if node.attributes.get("nocache"):
uc = 0
if not nx_attr:
recipe_error(rpstack, _("Missing %s attribute for %s")
% (attrnames[0], node.short_name()))
todolist.remove(node)
faillist.append(node)
elif ((not attr or nx_attr == attr)
and (this_use_cache is None or this_use_cache == uc)):
thislist.append(node)
todolist.remove(node)
attr = nx_attr
this_use_cache = uc
# Limit to 20 to avoid very long command lines.
if len(thislist) >= groupcount:
break
# Do the list of nodes with identical attributes "attr". There is at
# least one.
from Dictlist import str2dictlist
alist = str2dictlist(rpstack, attr)
if not alist:
recipe_error(rpstack, _("%s attribute for %s is empty")
% (attrname, thislist[0].short_name()))
# Perform the action, unless started with --nobuild or --touch.
if skip_commands():
if not Global.cmd_args.options.get("touch"):
msg_info(recdict, _('skip %s for %s') % (action,
str(map(lambda x: x.short_name(), thislist))))
else:
# Loop over the list of locations. Quit as soon as one worked,
# except for "publish", all locations are used then.
for loc in alist:
# Try handling thislist with this location. For publish do
# this for all locations, otherwise stop when all items have
# been done.
res = verscont_command(recdict, loc, thislist,
this_use_cache, action)
if action == "publish":
for r in res:
if not r in faillist:
faillist.append(r)
else:
thislist = res
if not thislist:
break
if action != "publish" and thislist:
# this bunch failed. Still continue doing the remaining ones.
faillist.extend(thislist)
return faillist
def fetch_nodelist(rpstack, recdict, nodelist, use_cache):
"""Fetch nodes in "nodelist" according to its "fetch" attribute.
When there is no "fetch" attribute use "commit".
Only use cached files when "use_cache" is non-zero.
Return list of failed nodes."""
return handle_nodelist(rpstack, recdict, nodelist, use_cache,
"fetch", [ "fetch", "commit" ])
def verscont_nodelist(rpstack, recdict, nodelist, action):
"""Checkout nodes in "nodelist" according to their "commit" attribute.
Return list of failed nodes."""
return handle_nodelist(rpstack, recdict, nodelist, 0, action, [ "commit" ])
def publish_nodelist(rpstack, recdict, nodelist, msg):
"""Publish nodes in "nodelist" according to their "publish" attribute.
When there is no "publish" attribute use "commit".
When "msg" is non-zero give a message for up-to-date nodes.
Return None for nothing done, list of failed nodes otherwise.
"""
# Publishing is only done when the signature is outdated.
# Need to pull a few tricks to be able to call check_need_update().
from Sign import sign_updated, buildcheck_updated
from DoBuild import Update, check_need_update
todolist = []
buildchecklist = []
targetlist = []
failed = []
for node in nodelist:
# Skip nodes that don't exist.
node_name = node.get_name()
if not os.path.exists(node_name):
failed.append(node)
msg_error(recdict,
_('Published file does not exist: "%s"') % node_name)
continue
if node.attributes.has_key("publish"):
s = node.attributes["publish"]
elif node.attributes.has_key("commit"):
s = node.attributes["commit"]
else:
s = ''
# Handle each publish/commit item separately, so that when adding a new
# destination we don't upload to the old destinations as well.
from Dictlist import str2dictlist, dictlist2str
alist = str2dictlist(rpstack, s)
if not alist:
recipe_error(rpstack, _('publish attribute for "%s" is empty')
% node.short_name())
for attr in alist:
# Use a fake virtual target to attach the signature to.
# Use the "remember" attribute, only publish when changed.
# Use the sign file in the directory related to the published file.
attr_name = attr["name"]
attr_str = dictlist2str([ attr ])
# Change "scp://" and "rscync://" to "scheme://", so that changing
# the method for uploading doesn't cause the signature to be
# outdated.
attr_name = re.sub("^[a-z]*://", "scheme://", attr_name)
buildcheck_str = re.sub("\\b[a-z]*://", "scheme://", attr_str)
target = Node(":publish:" + node_name + ">" + attr_name)
target.attributes["virtual"] = 1
target.attributes["remember"] = 1
target.attributes["signfile"] = node.get_sign_fname()
source = node.copy()
source.attributes["_node"] = source
source.attributes["name"] = node_name
source.attributes["publish"] = attr_str
# Need to publish the file when it's signature has changed or the
# relevant attributes have changed (use buildcheck for this).
update = Update()
check_need_update(recdict, update, source.attributes, target,
rootname = ":publish:" + node_name + ">")
update.set_buildcheck(recdict, buildcheck_str, target, 0)
# Find out if there is a reason to publish this source node.
reason = update.upd_reason(0, target)
if reason or msg:
disp_name = node.short_name() + "[" + attr_str + "]"
if reason:
if Global.cmd_args.options.get("touch"):
msg_info(recdict, _('Touching "%s": %s')
% (disp_name, reason))
else:
msg_depend(recdict, _('Publishing "%s": %s')
% (disp_name, reason))
todolist.append(source)
buildchecklist.append(update.buildcheck)
targetlist.append(target)
else:
if msg:
msg_depend(recdict, _('item is up-to-date: "%s"')
% disp_name)
# For the "--contents" option we skip items for which the
# "publish" attribute changed but not the contents. Need to
# update the signature anyway, otherwise it will still be
# published next time.
if (Global.cmd_args.options.get("contents")
and not Global.cmd_args.options.get("nobuild")):
sign_updated(recdict, source, source.attributes, target)
if update.buildcheck:
buildcheck_updated(recdict, target, update.buildcheck)
if not todolist:
if failed:
return failed # all outdated files do not exist
return None # all files exist and targets are up-to-date
failed.extend(handle_nodelist(rpstack, recdict, todolist, 0,
"publish", [ "publish", "commit" ]))
# Update signatures, unless --nobuild was used withouth --touch.
if (not Global.cmd_args.options.get("nobuild")
or Global.cmd_args.options.get("touch")):
i = 0
while i < len(todolist):
# Remember the source signature and the buildcheck signature.
# But only for nodes that didn't fail.
node = todolist[i]
if not node in failed:
target = targetlist[i]
buildcheck = buildchecklist[i]
sign_updated(recdict, node, node.attributes, target)
if buildcheck:
buildcheck_updated(recdict, target, buildcheck)
i = i + 1
return failed
def verscont_remove_add(rpstack, recdict, dir, recursive, action):
"""When "action" is "remove: Remove all files in directory "dir" of VCS
that don't belong there.
When "action" is "add:" Add all files in the recipe in directory "dir"
to the VCS that are missing.
"dir" is a dictionary for the directory and its attributes.
Enter directories recursively when "recursive" is non-zero.
"""
if not dir.has_key("commit"):
recipe_error(rpstack, _("no commit attribute for %s") % dir["name"])
from Dictlist import str2dictlist
commit_list = str2dictlist(rpstack, dir["commit"])
if not commit_list:
recipe_error(rpstack, _("commit attribute for %s is empty")
% dir["name"])
dirname = os.path.abspath(dir["name"])
alist = None
for commit_item in commit_list:
scheme, name = separate_scheme(commit_item["name"])
# Handle a scheme for which there is a scheme_list() function.
fun = recdict["_no"].get(scheme + "_list")
if fun:
alist = apply(fun, (recdict, name, commit_item, dirname, recursive))
break
# Handle the "cvs://" scheme.
if scheme == "cvs":
alist = cvs_list(recdict, name, commit_item, dirname, recursive)
break
if alist is None:
recipe_error(rpstack, _("No working item in commit attribute for %s")
% dir["name"])
work = getwork(recdict)
ok = 1
if action == "remove":
# Remove: Loop over all items found in the VCS.
for item in alist:
node = work.find_node(item)
if not node or not node.attributes.has_key("commit"):
if not node:
node = Node(item)
if skip_commands():
if not Global.cmd_args.options.get("touch"):
msg_info(recdict, _('Remove "%s"') % node.short_name())
elif verscont_command(recdict, commit_item, [ node ],
0, "remove"):
ok = 0
else:
# Add: Loop over all nodes and check if they are in the VCS.
# Only do this for a node that:
# - has the "commit" attribute
# - has a full name that is longer than the directory name
# - working recursive and the node is below the directory
# - not working recursive and the node is in the directory
# - the node does not appear in the list from the VCS
dirname_len = len(dirname)
for node in work.nodes.values():
if (node.attributes.has_key("commit")
and len(node.absname) > dirname_len
and ((recursive
and node.absname[:dirname_len] == dirname
and node.absname[dirname_len] == '/')
or (not recursive
and os.path.dirname(node.absname) == dirname))
and not node.absname in alist):
if skip_commands():
if not Global.cmd_args.options.get("touch"):
msg_info(recdict, _('Add "%s"') % node.short_name())
elif verscont_command(recdict, commit_item, [ node ], 0, "add"):
ok = 0
return ok
# vim: set sw=4 et sts=4 tw=79 fo+=l:
syntax highlighted by Code2HTML, v. 0.9.1