# Part of the A-A-P recipe executive: Aap commands in a recipe

# Copyright (C) 2002-2003 Stichting NLnet Labs
# Permission to copy and use this file is specified in the file COPYING.
# If this file is missing you can find it here: http://www.a-a-p.org/COPYING

#
# These are functions used to handle the commands in a recipe.
# Some functions are used for translated items, such as dependencies.
#
# It's OK to do "from Commands import *", these things are supposed to be
# global.
#

import os
import os.path
import sys

import import_re        # import the re module in a special way

import string
import copy
import glob
import imp

from Depend import Depend
from Dictlist import str2dictlist, get_attrdict, list2str, dictlist2str
from Dictlist import listitem2str
from Dictlist import dictlist_expanduser, dict_expand, dictlist_expand
from DoRead import read_recipe, recipe_dir, did_read_recipe
from DoArgs import doargs, add_cmdline_settings, copy_global_options
from Error import *
from Filetype import ft_known, ft_declare
import Global
from Process import assert_var_name, assert_scope_name
from Process import recipe_error, option_error
from RecPos import rpdeepcopy, rpcopy
from Rule import Rule
from Util import *
from Work import getwork, setwork, getrpstack, Route, set_defaults
from Work import assert_attribute
from Work import Work
from Message import *


def get_args(line_nr, recdict, arg, options = None, exp_attr = 1):
    """Parse command arguments.  Always use A-A-P style expansion.
       When "options" is given, it must be a dictionary indexed by accepted
       options with the value of the option name.
       "exp_attr" can be set to 0 to avoid including attributes.
       Return a dictionary with options, a dictionary with leading attributes
       and a dictlist with arguments."""
    # Get the optional leading attributes.
    rpstack = getrpstack(recdict, line_nr)
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    optiondict = {}

    # Move options from attrdict to optiondict.  Translate aliases.
    if options:
        for k in attrdict.keys():
            if options.has_key(k):
                optiondict[options[k]] = attrdict[k]
                del attrdict[k]

    # Get the list of items.  Expand $var things.
    varlist = str2dictlist(rpstack,
              expand(line_nr, recdict, arg[i:],
                                           Expand(exp_attr, Expand.quote_aap)))

    return optiondict, attrdict, varlist


def aap_pass(line_nr, recdict, arg):
    """
    Do nothing.
    """
    if arg:
        rpstack = getrpstack(recdict, line_nr)
        recipe_error(rpstack, _(':pass does not take an argument'))


def aap_buildcheck(line_nr, recdict, arg):
    """
    Do nothing with an argument.  Used to mention variables that should be
    included in the buildcheck.
    """
    pass


def aap_depend(line_nr, recdict, targets, sources, commands):
    """Add a dependency."""
    work = getwork(recdict)
    rpstack = getrpstack(recdict, line_nr)

    # Expand the targets into dictlists.
    targetlist = str2dictlist(rpstack,
                expand(line_nr, recdict, targets, Expand(1, Expand.quote_aap)))
    targetlist = dictlist_expand(targetlist)

    # Parse build attributes {attr = value} zero or more times.
    # Variables are not expanded now but when executing the build rules.
    build_attr, i = get_attrdict(rpstack, None, sources, 0, 0)

    # Expand the sources into dictlists.
    sourcelist = str2dictlist(rpstack,
            expand(line_nr, recdict, sources[i:], Expand(1, Expand.quote_aap)))
    sourcelist = dictlist_expand(sourcelist)

    # Add to the global lists of dependencies.
    # Make a copy of the RecPos stack, so that we can print the recipe stack
    # for an error.  The parsing continues, thus it needs to be a copy.
    d = Depend(targetlist, build_attr, sourcelist, work,
                        rpdeepcopy(getrpstack(recdict), line_nr), commands)

    # We need to remember the recdict of the recipe where the dependency was
    # defined.
    d.buildrecdict = recdict

    work.add_dependency(rpstack, d)


def aap_clearrules(line_nr, recdict, arg):
    """Generate a recipe for downloading files."""
    rpstack = getrpstack(recdict, line_nr)
    work = getwork(recdict)

    if arg:
        recipe_error(rpstack, _(':clearrules does not take an argument'))
    work.clearrules()


def aap_delrule(line_nr, recdict, targets, sources):
    """Delete a rule."""
    aap_rule(line_nr, recdict, targets, sources, None, cmd = ":delrule")

def aap_rule(line_nr, recdict, targets, sources, commands, cmd = ":rule"):
    """Add a rule."""
    work = getwork(recdict)
    rpstack = getrpstack(recdict, line_nr)

    # Parse command attributes {global} or {local} zero or more times.
    scope = "normal"
    default = 0
    sourceexists = 0
    quiet = 0
    cmd_attr, i = get_attrdict(rpstack, None, targets, 0, 0)

    for k in cmd_attr.keys():
        if cmd == ":delrule" and (k == "quiet" or k == "q"):
            quiet = 1
        elif cmd == ":rule" and (k == "local" or k == "global"):
            if scope != "normal":
                recipe_error(rpstack, _('extra option for :rule: "%s"') % k)
            scope = k
        elif cmd == ":rule" and k == "sourceexists":
            sourceexists = 1
        elif cmd == ":rule" and k == "default":
            default = 1
        else:
            recipe_error(rpstack, _('unknown option for %s: "%s"') % (cmd, k))

    # Expand the targets into dictlists.
    targetlist = str2dictlist(rpstack,
            expand(line_nr, recdict, targets[i:], Expand(1, Expand.quote_aap)))
    dictlist_expanduser(targetlist)

    # Parse any build attributes {attr = value}.
    # Variables in the attributes are not expanded now but when executing the
    # build rules.
    build_attr, i = get_attrdict(rpstack, None, sources, 0, 0)

    # Expand the sources into dictlists.
    sourcelist = str2dictlist(rpstack,
          expand(line_nr, recdict, sources[i:], Expand(1, Expand.quote_aap)))
    dictlist_expanduser(sourcelist)

    # Check if there is an identical rule.
    did_one = 0
    for r in work.rules:
        if (dictlist_sameentries(targetlist, r.targetlist)
                and dictlist_sameentries(sourcelist, r.sourcelist)):
            if cmd == ":rule" and not r.default:
                msg_warning(recdict, _("Replacing existing rule"),
                                                             rpstack = rpstack)
            work.del_rule(r)
            did_one = 1

    if cmd == ":delrule":
        if not did_one and not quiet:
            msg_warning(recdict, _("Did not find matching rule to delete"),
                                                             rpstack = rpstack)
    else:
        rule = Rule(targetlist, build_attr, sourcelist,
                        rpdeepcopy(getrpstack(recdict), line_nr), commands) 
        rule.default = default
        rule.sourceexists = sourceexists
        rule.scope = scope

        # We need to remember the recdict of the recipe where the rule was
        # defined.
        rule.buildrecdict = recdict

        work.add_rule(rule)


def aap_route(line_nr, recdict, arg, linestring):
    """Add a route."""
    work = getwork(recdict)
    rpstack = getrpstack(recdict, line_nr)

    # Parse command attributes {global} or {local} zero or more times.
    default = 0
    cmd_attr, i = get_attrdict(rpstack, None, arg, 0, 0)
    for k in cmd_attr.keys():
        if k == "default":
            default = 1
        else:
            recipe_error(rpstack, _('unknown option for :route: "%s"') % k)

    # Expand the list of filetypes into dictlists.
    arglist = str2dictlist(rpstack,
            expand(line_nr, recdict, arg[i:], Expand(1, Expand.quote_aap)))
    typelist = map(lambda x: x["name"], arglist)
    if len(typelist) < 2:
        recipe_error(rpstack, _(':route: requires at least two filetypes'))

    # Process the action lines.
    # Skip lines starting with a comment.
    # Concatenate lines that have more indent than the first line.
    from Process import get_line_marker

    rawlines = string.split(linestring, '\n')
    first_indent = -1
    lines = []
    lnums = []
    lnum = line_nr + 1
    for line in rawlines:
        i = skip_white(line, 0)
        if i < len(line):
            if line[i] == '#':
                nr = get_line_marker(line[i:])
                if nr:
                    lnum = nr
            else:
                ind = get_indent(line)
                if first_indent == -1:
                    first_indent = ind
                if ind > first_indent:
                    lines[-1] = lines[-1] + line    # append to previous line
                else:
                    lines.append(line)              # append action line
                    lnums.append(lnum)

    if len(typelist) != len(lines) + 1:
        recipe_error(rpstack, _(':route: found %d action lines, expected %d')
                % (len(lines), len(typelist) - 1))

    # Each entry in typelist can be a comma separate list of filetypes.  Turn
    # it into a list of lists.
    typelist = map(lambda x: string.split(x, ","), typelist)

    # Check that all the filetypes are known
    for il in typelist:
	for i in il:
            if not ft_known(i):
                msg_warning(recdict, _('unknown filetype "%s" in :route command; recipe %s line %d') % (i, rpstack[-1].name, rpstack[-1].line_nr))

    # Now add the routes themselves
    for in_type in typelist[0]:
        for out_type in typelist[-1]:
            route = work.find_route(in_type, out_type)
            if route and not route.default:
                recipe_error(rpstack,
                        _('redefining existing route from "%s" to "%s"')
                                                         % (in_type, out_type))

    route = Route(recdict, rpdeepcopy(rpstack, line_nr), default,
                                           typelist, arglist[-1], lines, lnums)
    work.add_route(route)


def aap_program(line_nr, recdict, arg, type = "program"):
    """
    Add a program, lib, dll, etc. build from sources.
    Also for ":totype".
    "arg" is the whole argument string, except for ":produce" where it lacks
    the first argument (turned into the type).
    """
    rpstack = getrpstack(recdict, line_nr)
    work = getwork(recdict)

    # Parse command attributes.
    cmd_attr, i = get_attrdict(rpstack, recdict, arg, 0, 1)

    # Expand the arguments into a dictlist.
    dictlist = str2dictlist(rpstack,
                expand(line_nr, recdict, arg[i:], Expand(1, Expand.quote_aap)))

    cmdname = type

    # For ":produce" the first argument is the type of the target.
    atype = type
    if atype == "produce":
        if len(dictlist) < 2:
            recipe_error(rpstack,
                     _(":produce requires both a type name and a target name"))
        atype = dictlist[0]["name"]
        cmd_attr.update(dictlist[0])
        dictlist = dictlist[1:]

    # Find the ":" item between the target and the sources.
    colonidx = -1
    for i in range(len(dictlist)):
        if dictlist[i]["name"] == ":":
            colonidx = i
            break
    if colonidx == -1:
        recipe_error(rpstack, _("Missing ':' after :%s") % cmdname)

    targetlist = dictlist[0:colonidx]
    if atype != "totype":
        targetlist = dictlist_expand(targetlist)
    if len(targetlist) != 1:
        recipe_error(rpstack, _(":%s requires one target") % cmdname)

    # get any build attributes {attr = value} from after the ':'.
    build_attr = dictlist[colonidx]

    # Expand the sources into dictlists.
    sourcelist = dictlist[colonidx + 1:]
    sourcelist = dictlist_expand(sourcelist)
    if len(sourcelist) < 1:
        recipe_error(rpstack, _(":%s requires at least one source") % cmdname)

    # If there are build attributes, apply them to all the
    # source files.
    for ba in build_attr.keys():
        # Everything is named already, skip it
        if ba == "name":
            continue
        value = build_attr[ba]
        if ba.startswith("add_") or ba.startswith("var_"):
            # Build var_ + source var_ => source takes precedence
            # Build add_ + source var_ => source takes precedence
            # Build var_ + source add_ => append source to build
            # Build add_ + source add_ => append source to build ?
            # Build var_ + no source   => use build var
            # Build add_ + no source   => use build add
            varname = ba[4:]
            for source in sourcelist:
                # Source-level var_ attributes take precedence
                if source.has_key("var_"+varname):
                    continue
                # Prepend build attribute
                if source.has_key("add_"+varname):
                    if ba.startswith("add_"):
                        source[ba] = value + " " + source[ba]
                    else:
		        source["var_"+varname]=value + " " + source["add_"+varname]
			del source["add_"+varname]
                else:
                    source[ba] = value
        else:
            for source in sourcelist:
                if not source.has_key(ba):
                    source[ba] = value

    # ":produce abc" needs to declare "abc" as a filetype.
    if cmdname == "produce":
        ft_declare(atype)

    if atype != "totype":
        # If the target doesn't have an suffix, add $EXESUF.  It's done here so
        # that the user can set $EXESUF before/after ":program".
        bn = os.path.basename(targetlist[0]["name"])
        if string.find(bn, ".") < 0:
            # Remember the original name as an alias, it may be used as a
            # source in another dependency (e.g., for "all").
            alias = targetlist[0]["name"]

            if atype == "program":
                prevar = None
                sufvar = "EXESUF"
            elif atype == "lib":
                prevar = "LIBPRE"
                sufvar = "LIBSUF"
            elif atype == "ltlib":
                prevar = "LTLIBPRE"
                sufvar = "LTLIBSUF"
            elif atype == "dll":
                prevar = "DLLPRE"
                sufvar = "DLLSUF"
            else:  # ":produce"
                prevar = None
                sufvar = None

            # Get the prefix from a variable or from the "prefix" attribute.
            pre = cmd_attr.get("targetprefix")
            if not pre and prevar:
                pre = get_var_val(line_nr, recdict, "_no", prevar)
            if pre:
                newname = alias[:-len(bn)] + pre + bn
            else:
                newname = alias

            # Get the suffix from a variable or from the "suffix" attribute.
            suf = cmd_attr.get("targetsuffix")
            if not suf and sufvar:
                suf = get_var_val(line_nr, recdict, "_no", sufvar)
            if suf:
                newname = newname + suf
            targetlist[0]["name"] = newname

            # Add the alias to the node.  May rename an existing node.
            work.node_set_alias(newname, alias, targetlist[0])

            # Add a comment attribute if there isn't one.
            if not targetlist[0].get("comment"):
                s = cmd_attr.get("comment")
                if not s:
                    d = {"program" : "program",
                               "lib" : "static library",
                               "ltlib" : "libool archive",
                               "dll" : "shared library" }
                    s = d.get(atype)
                if not s:
                    s = atype
                targetlist[0]["comment"] = (_('build %s "%s"') % (s, newname))

    # Setting the "filetype" attribute on the target would overrule a filetype
    # that the user has given to the node.  Use "filetypehint" to avoid that.
    targetlist[0]["filetypehint"] = atype

    # Add the build rule.
    from DoAddDef import add_buildrule
    add_buildrule(rpstack, work, recdict, 
                            atype, cmd_attr, targetlist, build_attr, sourcelist)

    if atype != "totype":
        # Add the target to the default targets.
        t = targetlist[0]
        recdict["_buildrule_targets"].append(work.get_node(t["name"], 0, t))


def aap_lib(line_nr, recdict, arg):
    """Add a static library built from sources."""
    aap_program(line_nr, recdict, arg, type = "lib")


def aap_ltlib(line_nr, recdict, arg):
    """Add a libtool library built from sources."""
    # We automatically import the libtool module.
    if not getwork(recdict).module_already_read.has_key("libtool"):
        aap_import(line_nr, recdict, "libtool")
    aap_program(line_nr, recdict, arg, type = "ltlib")

def aap_dll(line_nr, recdict, arg):
    """Add a dynamic library built from sources."""
    aap_program(line_nr, recdict, arg, type = "dll")


def aap_produce(line_nr, recdict, arg):
    """Produce something user-defined from sources."""
    aap_program(line_nr, recdict, arg, type = "produce")


def aap_totype(line_nr, recdict, arg):
    """Add a dynamic library build from sources."""
    aap_program(line_nr, recdict, arg, type = "totype")


def aap_tree(line_nr, recdict, arg, cmd_line_nr, commands):
    """":tree"."""
    rpstack = getrpstack(recdict, line_nr)

    # Expand dirname and options into a dictlist
    dictlist = str2dictlist(rpstack,
                    expand(line_nr, recdict, arg, Expand(1, Expand.quote_aap)))
    if not dictlist[0]["name"]:
        # Arguments start with an attribute.
        recipe_error(rpstack, _('First argument of :tree must be a directory name'))
    dictlist_expanduser(dictlist)
    if len(dictlist) != 1:
        recipe_error(rpstack, _(":tree requires one directory name"))

    for k in dictlist[0].keys():
        if k not in [ "filename", "dirname", "reject", "skipdir",
                                                "contents", "name", "follow" ]:
            recipe_error(rpstack, _('Invalid attribute for :tree: %s') % k)

    dirpat = dictlist[0].get("dirname")
    filepat = dictlist[0].get("filename")
    if not dirpat and not filepat:
        # No pattern specified, match all files
        filepat = ".*"

    # Compile the pattern.  Prepend "^(" and add ")$" to match the whole name
    # and allow alternatives: "^(" + "foo|bar" + ")$".
    # Ignore case on non-posix systems
    if os.name == "posix":
        flags = 0
    else:
        flags = re.IGNORECASE
    options = {}
    if dirpat:
        try:
            options["dirrex"] = re.compile("^(" + dirpat + ")$", flags)
        except Exception, e:
            recipe_error(rpstack, _('Invalid pattern "%s": %s')
                                                            % (dirpat, str(e)))
    else:
        options["dirrex"] = None
    if filepat:
        try:
            options["filerex"] = re.compile("^(" + filepat + ")$", flags)
        except Exception, e:
            recipe_error(rpstack, _('Invalid pattern "%s": %s')
                                                           % (filepat, str(e)))
    else:
        options["filerex"] = None

    # Compile the reject pattern if given.
    rejectpat = dictlist[0].get("reject")
    if rejectpat:
        try:
            options["rejectrex"] = re.compile("^(" + rejectpat + ")$", flags)
        except Exception, e:
            recipe_error(rpstack, _('Invalid pattern "%s": %s')
                                                         % (rejectpat, str(e)))
    else:
        options["rejectrex"] = None

    # Compile the skipdir pattern if given.
    skipdirpat = dictlist[0].get("skipdir")
    if skipdirpat:
        try:
            options["skipdirrex"] = re.compile("^(" + skipdirpat + ")$", flags)
        except Exception, e:
            recipe_error(rpstack, _('Invalid pattern "%s": %s')
                                                         % (skipdirpat, str(e)))
    else:
        options["skipdirrex"] = None

    # Compile the text contents pattern if given.
    contentspat = dictlist[0].get("contents")
    if contentspat:
        try:
            options["contentsrex"] = re.compile(contentspat)
        except Exception, e:
            recipe_error(rpstack, _('Invalid pattern "%s": %s')
                                                       % (contentspat, str(e)))
    else:
        options["contentsrex"] = None

    # Take over the "follow" option.
    options["follow"] = dictlist[0].get("follow")

    aap_tree_recurse(rpstack, recdict, dictlist[0]["name"], options,
                                                         cmd_line_nr, commands)

def aap_tree_recurse(rpstack, recdict, dir, options, cmd_line_nr, commands):
    """
    Handle one directory level for ":tree" and recurse into subdirectories.
    """
    from ParsePos import ParsePos
    from Process import Process

    # Handle the situation that a directory is not readable.
    try:
        dirlist = os.listdir(dir)
    except:
        msg_warning(recdict, _('Cannot read directory "%s"') % dir)
        return

    for f in dirlist:
        f = os.path.join(dir, f)

        # Handle recursion, but don't follow links and skip directories that
        # match the "skipdir" pattern.
        srex = options["skipdirrex"]
        if (os.path.isdir(f)
                and (options["follow"] or not os.path.islink(f))
                and (not srex or not srex.match(os.path.basename(f)))):
            aap_tree_recurse(rpstack, recdict, f, options,
                                                         cmd_line_nr, commands)

        if os.path.isfile(f):
            rex = options["filerex"]
        elif os.path.isdir(f):
            rex = options["dirrex"]
        else:
            rex = None
        rrex = options["rejectrex"]
        crex = options["contentsrex"]
        n = os.path.basename(f)
        if rex and rex.match(n) and (not rrex or not rrex.match(n)):
            if crex and os.path.isfile(f):
                try:
                    fd = open(f)
                except:
                    msg_warning(recdict, _('Cannot open "%s"') % f)
                    continue
                match = 0
                try:
                    while 1:
                        line = fd.readline()
                        if not line:
                            break
                        if crex.match(line):
                            match = 1
                            break
                except:
                    fd.close()
                    msg_warning(recdict, _('Cannot read "%s"') % f)
                    continue
                fd.close()
                if not match:
                    continue

            # Use the current scope for the the command block.  This makes it
            # easier to use, like a "for" loop.
            new_rpstack = rpcopy(rpstack, cmd_line_nr)
            name_save = recdict.get("name")
            recdict["name"] = listitem2str(f)

            # Create a ParsePos object to contain the parse position in the
            # string.
            # Parse and execute the commands.
            fp = ParsePos(new_rpstack, string = commands)
            Process(fp, recdict, 0)

            if name_save is None:
                del recdict["name"]
            else:
                recdict["name"] = name_save


def aap_update(line_nr, recdict, arg):
    """Handle ":update target ...": update target(s) now."""
    work = getwork(recdict)
    rpstack = getrpstack(recdict, line_nr)
    from DoBuild import target_update, updated_OK

    # Get the arguments and check the options.
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"f": "force", "force" : "force",
             "searchpath" : "searchpath" })
    if attrdict:
        option_error(rpstack, attrdict, ":update")

    if len(argdictlist) == 0:
        recipe_error(rpstack, _("Missing argument for :update"))

    argdictlist = dictlist_expand(argdictlist)

    adir = os.getcwd()

    # Loop over all arguments.
    for t in argdictlist:
        msg_depend(recdict, _('Start :update "%s"') % t["name"])

        if os.path.isabs(t["name"]):
            sp = [ '' ]         # do not use "searchpath" for absolute name
        else:
            sp = optiondict.get("searchpath")
            if not sp:
                sp = [ '' ]     # No {searchpath = ...}, do not use a path.
            else:
                sp = map(lambda x: x["name"], str2dictlist(rpstack, sp))

        # Loop over "searchpath", the first one that works is used.
        done = 0
        for path in sp:
            name = t["name"]
            if path:
                name = os.path.join(path, name)
            target = work.get_node(name)

            if len(sp) > 1:
                msg_depend(recdict, _('Attempt :update "%s"') % name)

            # For finding rules the scope of the current recipe is used.
            target.scope_recdict = recdict

            if target_update(work, recdict, target, 0,
                             optiondict.get("force"), level = 1) == updated_OK:
                done = 1
                if len(sp) > 1:
                    msg_depend(recdict, _(':update "%s" successful') % name)
                break

            if len(sp) > 1:
                msg_depend(recdict, _(':update "%s" failed') % name)

        if not done:
            recipe_error(rpstack, _(':update failed for "%s"') % t["name"])

        msg_depend(recdict, _('End of :update "%s"') % t["name"])

    # Return to the original directory, could be halfway build commands.
    goto_dir(recdict, adir)


def aap_error(line_nr, recdict, arg):
    """Handle: ":error foo bar"."""
    rpstack = getrpstack(recdict, line_nr)
    recipe_error(rpstack, expand(line_nr, recdict, arg,
                                                 Expand(1, Expand.quote_aap)))

def aap_unknown(line_nr, recdict, arg):
    """
    Handle: ":xxx arg".  Postponed until executing the line, so that an "@if
    _no.AAPVERSION > nr" can be used.
    """
    rpstack = getrpstack(recdict, line_nr)
    recipe_error(rpstack, _('Unknown command: "%s"') % arg)

def aap_nothere(line_nr, recdict, arg):
    """
    Handle using a toplevel command in build commands.  Postponed until
    executing the line, so that an "@if _no.AAPVERSION > nr" can be used.
    """
    rpstack = getrpstack(recdict, line_nr)
    recipe_error(rpstack, _('Command cannot be used here: "%s"') % arg)

#
############## start of commands used in a pipe
#

def _get_redir(line_nr, recdict, raw_arg, xp):
    """Get the redirection and pipe from the argument "raw_arg".
       Returns these items:
       1. the argument with $VAR expanded and redirection removed
       2. the file name for redirection or None
       3. the mode for redirection or None ("a" for append, "w" for write).
       4. a command following '|' or None
       When using ">file" also checks if the file doesn't exist yet.
       Use "xp" for expanding $var in the argument."""
    rpstack = getrpstack(recdict, line_nr)

    mode = None
    fname = None
    nextcmd = None

    # Loop over the argument, getting one token at a time.  Each token is
    # either non-white (possibly with quoting) or a span of white space.
    raw_arg_len = len(raw_arg)
    i = 0           # current index in raw_arg
    new_arg = ''    # argument without redirection so far
    while i < raw_arg_len:
        t, i = get_token(raw_arg, i)

        # Ignore trailing white space.
        if i == raw_arg_len and is_white(t[0]):
            break

        # After (a span of) white space, check for redirection or pipe.
        # Also at the start of the argument.
        if new_arg == '' or is_white(t[0]):
            if new_arg == '':
                # Use the token at the start of the argument.
                nt = t
                t = ''
            else:
                # Get the token after the white space
                nt, i = get_token(raw_arg, i)

            if nt[0] == '>':
                # Redirection: >, >> or >!
                if mode:
                    recipe_error(rpstack, _('redirection appears twice'))
                nt_len = len(nt)
                ni = 1      # index after the ">", ">>" or ">!"
                mode = 'w'
                overwrite = 0
                if nt_len > 1:
                    if nt[1] == '>':
                        mode = 'a'
                        ni = 2
                    elif nt[1] == '!':
                        overwrite = 1
                        ni = 2
                if ni >= nt_len:
                    # white space after ">", get next token for fname
                    redir = nt[:ni]
                    if i < raw_arg_len:
                        # Get the separating white space.
                        nt, i = get_token(raw_arg, i)
                    if i == raw_arg_len:
                        recipe_error(rpstack, _('Missing file name after %s')
                                                                       % redir)
                    # Get the file name
                    nt, i = get_token(raw_arg, i)
                else:
                    # fname follows immediately after ">"
                    nt = nt[ni:]

                # Expand $VAR in the file name.  No attributes are added.
                # Remove quotes from the result, it's used as a filename.
                fname = unquote(recdict, expand(line_nr, recdict, nt,
                                                  Expand(0, Expand.quote_aap)))
                from Remote import url_split3
                scheme, machine, path = url_split3(fname)
                if not scheme:
                    # Not an URL: expand wildcards and check for overwriting.
                    l = glob.glob(os.path.expanduser(fname))
                    if len(l) == 1:
                        fname = l[0]
                    # else: give an error message?
                    if mode == "w" and not overwrite:
                        check_exists(rpstack, fname)
                else:
                    # No expanding in a URL yet, at least remove escaped
                    # wildcards: [*] -> *.
                    fname = expand_unescape(fname)

                # When redirection is at the start, ignore the white space
                # after it.
                if new_arg == '' and i < raw_arg_len:
                    t, i = get_token(raw_arg, i)

            elif nt[0] == '|':
                # Pipe: the rest of the argument is another command
                if mode:
                    recipe_error(rpstack, _("both redirection and '|'"))

                if len(nt) > 1:
                    nextcmd = nt[1:] + raw_arg[i:]
                else:
                    i = skip_white(raw_arg, i)
                    nextcmd = raw_arg[i:]
                if not nextcmd:
                    # Can't have an empty command.
                    recipe_error(rpstack, _("Nothing follows '|'"))
                if nextcmd[0] != ':':
                    # Must have an aap command here.
                    recipe_error(rpstack, _("Missing ':' after '|'"))
                break

            else:
                # No redirection or pipe: add to argument
                new_arg = new_arg + t + nt
        else:
            # Normal token: add to argument
            new_arg = new_arg + t

    if new_arg and xp:
        arg = expand(line_nr, recdict, new_arg, xp)
    else:
        arg = new_arg

    return arg, fname, mode, nextcmd


def _aap_pipe(line_nr, recdict, cmd, pipein):
    """Handle the command that follows a '|'."""
    rpstack = getrpstack(recdict, line_nr)
    items = string.split(cmd, None, 1)
    if len(items) == 1:     # command without argument, append empty argument.
        items.append('')

    if items[0] == ":assign":
        _pipe_assign(line_nr, recdict, items[1], pipein)
    elif items[0] == ":cat":
        aap_cat(line_nr, recdict, items[1], pipein)
    elif items[0] == ":syseval":
        aap_syseval(line_nr, recdict, items[1], pipein)
    elif items[0] == ":eval":
        aap_eval(line_nr, recdict, items[1], pipein)
    elif items[0] == ":print":
        aap_print(line_nr, recdict, items[1], pipein)
    elif items[0] == ":tee":
        _pipe_tee(line_nr, recdict, items[1], pipein)
    else:
        recipe_error(rpstack,
                           (_('Invalid command after \'|\': "%s"') % items[0]))


def _pipe_assign(line_nr, recdict, raw_arg, pipein):
    """Handle: ":assign var".  Can only be used in a pipe."""
    rpstack = getrpstack(recdict, line_nr)
    assert_var_name(rpstack, raw_arg)

    # Separate scope and decide which dictionary to use.
    rd, scope, varname = get_scope_recdict(rpstack, recdict, raw_arg, 1)
    rd[varname] = pipein


def aap_cat(line_nr, recdict, raw_arg, pipein = None):
    """Handle: ":cat >foo $bar"."""

    rpstack = getrpstack(recdict, line_nr)

    # get the special items out of the argument
    arg, fname, mode, nextcmd = _get_redir(line_nr, recdict, raw_arg,
                                                   Expand(0, Expand.quote_aap))

    # Skip writing file when not actually building.
    if mode and skip_commands():
        msg_skip(line_nr, recdict, ':cat ' + raw_arg)
        return

    # get the list of files from the remaining argument
    filelist = str2dictlist(rpstack, arg)
    if len(filelist) == 0:
        if pipein is None:
            recipe_error(rpstack,
                    _(':cat command requires at least one file name argument'))
        filelist = [ {"name" : "-"} ]

    result = ''
    scheme = None
    if mode:
        # Open the output file for writing
        from Remote import url_split3
        from RecPython import tempfname

        scheme, machine, path = url_split3(fname)
        if scheme:
            # Remote file, first write to a temp file.
            ffname = tempfname()
        else:
            ffname = fname
        try:
            wf = open(ffname, mode)
        except IOError, e:
            recipe_error(rpstack,
                         (_('Cannot open "%s" for writing') % ffname) + str(e))

    try:
        # Loop over all arguments
        filelist = dictlist_expand(filelist)
        for item in filelist:
            fn = item["name"]
            if fn == '-':
                # "-" argument: use pipe input
                if pipein is None:
                    recipe_error(rpstack, _('Using - not after a pipe'))
                if nextcmd:
                    result = result + pipein
                else:
                    # Split into separate lines; can't use split() here, we
                    # want to keep the line breaks.
                    lines = []
                    i = 0
                    while i < len(pipein):
                        n = string.find(pipein, "\n", i)
                        if n < 0:
                            lines.append(pipein[i:])
                            break
                        lines.append(pipein[i:n + 1])
                        i = n + 1
            else:
                # file name argument: read the file
                try:
                    rf = open(fn, "r")
                except IOError, e:
                    recipe_error(rpstack,
                             (_('Cannot open "%s" for reading') % fn) + str(e))
                try:
                    lines = rf.readlines()
                    rf.close()
                except IOError, e:
                    recipe_error(rpstack,
                                    (_('Cannot read from "%s"') % fn) + str(e))
                if nextcmd:
                    # pipe output: append lines to the result
                    for l in lines:
                        result = result + l

            if mode:
                # file output: write lines to the file
                try:
                    wf.writelines(lines)
                except IOError, e:
                    recipe_error(rpstack,
                                 (_('Cannot write to "%s"') % ffname) + str(e))
            elif not nextcmd:
                # output to the terminal: print lines
                for line in lines:
                    if line[-1] == '\n':
                        msg_print(line[:-1])
                    else:
                        msg_print(line)

        if mode:
            # close the output file
            try:
                wf.close()
            except IOError, e:
                recipe_error(rpstack, (_('Error closing "%s"')
                                                            % ffname) + str(e))
            else:
                if scheme:
                    from CopyMove import remote_copy_move
                    remote_copy_move(rpstack, recdict, 0,
                                     [ {"name" : ffname} ], { "name" : fname },
                                                             {}, 0, errmsg = 1)

    finally:
        if scheme:
            # Always delete the temp file.
            try_delete(ffname)

    if nextcmd:
        # pipe output: execute the following command
        _aap_pipe(line_nr, recdict, nextcmd, result)
    elif mode:
        msg_info(recdict, _('Concatenated files into "%s"') % fname)


def aap_eval(line_nr, recdict, raw_arg, pipein = None):
    """Handle: ":eval function ...".  Can be used in a pipe."""
    rpstack = getrpstack(recdict, line_nr)
    arg, fname, mode, nextcmd = _get_redir(line_nr, recdict, raw_arg, None)

    if pipein and string.find(arg, "stdin") < 0:
        recipe_error(rpstack, _('stdin missing in :eval argument'))

    # Evaluate the expression.
    if not pipein is None:
        recdict["stdin"] = pipein
    try:
        result = str(eval(arg, recdict, recdict))
    except StandardError, e:
        recipe_error(rpstack, (_(':eval command "%s" failed: ') % arg)
                                                                      + str(e))
    if pipein:
        del recdict["stdin"]

    if mode:
        # redirection: write output to a file
        _write2file(rpstack, recdict, fname, result, mode)
    elif nextcmd:
        # pipe output: execute next command
        _aap_pipe(line_nr, recdict, nextcmd, result)
    else:
        # output to terminal: print the result
        msg_print(result)


def aap_print(line_nr, recdict, raw_arg, pipein = None):
    """
    Handle: ":print foo $bar", print a message
    """
    rpstack = getrpstack(recdict, line_nr)
    if pipein:
        recdict["stdin"] = pipein
    arg, fname, mode, nextcmd = _get_redir(line_nr, recdict, raw_arg,
                                                   Expand(1, Expand.quote_aap))
    if pipein:
        del recdict["stdin"]
        if not arg:
            arg = pipein

    if mode:
        if len(arg) == 0 or arg[-1] != '\n':
            arg = arg + '\n'
        _write2file(rpstack, recdict, fname, arg, mode)
    elif nextcmd:
        if len(arg) == 0 or arg[-1] != '\n':
            arg = arg + '\n'
        _aap_pipe(line_nr, recdict, nextcmd, arg)
    else:
	msg_print(arg)


def aap_log(line_nr, recdict, raw_arg, pipein = None):
    """
    Handle: ":log foo $bar", write a message in the log file.
    """
    rpstack = getrpstack(recdict, line_nr)
    if pipein:
        recdict["stdin"] = pipein
    arg, fname, mode, nextcmd = _get_redir(line_nr, recdict, raw_arg,
                                                   Expand(1, Expand.quote_aap))
    if pipein:
        del recdict["stdin"]
        if not arg:
            arg = pipein

    if mode or nextcmd:
        recipe_error(rpstack, _('Cannot redirect or pipe :log output'))

    msg_log(recdict, arg)


def aap_syseval(line_nr, recdict, raw_arg, pipein = None):
    """Execute a shell command, redirecting stdout and/or stderr and stdin."""
    rpstack = getrpstack(recdict, line_nr)
    attrdict, idx = get_attrdict(rpstack, recdict, raw_arg, 0, 1)
    for k in attrdict.keys():
        if k != "stderr":
            recipe_error(rpstack, _('unknown option for :syseval: "%s"') % k)

    arg, fname, mode, nextcmd = _get_redir(line_nr, recdict, raw_arg[idx:],
                                                   Expand(1, Expand.quote_aap))

    try:
        tmpin = None
        tmpout = None

        from RecPython import tempfname
        cmd = '(' + arg + ')'
        if pipein:
            tmpin = tempfname()
            try:
                f = open(tmpin, "w")
                f.write(pipein)
                f.close()
            except IOError, e:
                recipe_error(rpstack, (_('Cannot write stdin to "%s": ')
                                                             % tmpin) + str(e))
            cmd = cmd + (' < %s' % tmpin)

        tmpout = tempfname()
        if attrdict.has_key("stderr"):
            cmd = cmd + ' 2>&1'
        cmd = cmd + (' > %s' % tmpout)

        recdict["exit"] = os.system(cmd)

        # read the output file
        try:
            rf = open(tmpout)
            out = rf.read()
            rf.close()
        except IOError:
            out = ''
    finally:
        # Clean up the temp files
        if tmpin:
            try_delete(tmpin)
        if tmpout:
            try_delete(tmpout)

    # Remove leading and trailing blanks and newlines.
    out = re.sub(r'^\s*', '', out)
    out = re.sub(r'\s*$', '', out)

    if mode:
        _write2file(rpstack, recdict, fname, out, mode)
    elif nextcmd:
        _aap_pipe(line_nr, recdict, nextcmd, out)
    else:
        msg_print(out)


def _pipe_tee(line_nr, recdict, raw_arg, pipein):
    """Handle: ":tee fname ...".  Can only be used in a pipe."""
    rpstack = getrpstack(recdict, line_nr)
    arg, fname, mode, nextcmd = _get_redir(line_nr, recdict, raw_arg,
                                                   Expand(0, Expand.quote_aap))

    # get the list of files from the remaining argument
    filelist = str2dictlist(rpstack, arg)
    if len(filelist) == 0:
        recipe_error(rpstack,
                    _(':tee command requires at least one file name argument'))

    for f in filelist:
        fn = f["name"]
        check_exists(rpstack, fn)
        _write2file(rpstack, recdict, fn, pipein, "w")

    if mode:
        # redirection: write output to a file
        _write2file(rpstack, recdict, fname, pipein, mode)
    elif nextcmd:
        # pipe output: execute next command
        _aap_pipe(line_nr, recdict, nextcmd, pipein)
    else:
        # output to terminal: print the result
        msg_print(pipein)


def _write2file(rpstack, recdict, fname, string, mode):
    """Write string "string" to file "fname" opened with mode "mode"."""
    # Skip when not actually building.
    if skip_commands():
        msg_info(recdict, _('skip writing to "%s"') % fname)
        return

    from Remote import url_split3
    from RecPython import tempfname

    scheme, machine, path = url_split3(fname)
    if scheme:
        # Remote file, first write to a temp file.
        ffname = tempfname()
    else:
        ffname = fname

    try:
        f = open(ffname, mode)
    except IOError, e:
        recipe_error(rpstack,
                         (_('Cannot open "%s" for writing') % ffname) + str(e))
    try:
        try:
            f.write(string)
            f.close()
        except IOError, e:
            recipe_error(rpstack, (_('Cannot write to "%s"') % fname) + str(e))
        
        if scheme:
            from CopyMove import remote_copy_move
            remote_copy_move(rpstack, recdict, 0,
                                     [ {"name" : ffname} ], { "name" : fname },
                                                             {}, 0, errmsg = 1)
    finally:
        if scheme:
            # Always delete the temp file.
            try_delete(ffname)

#
############## end of commands used in a pipe
#

def aap_child(line_nr, recdict, arg):
    """Handle ":child filename": read a recipe."""
    aap_child_and_exe(line_nr, recdict, arg, 1)

def aap_execute(line_nr, recdict, arg):
    """Handle ":execute filename [target] ...": execute a recipe."""
    aap_child_and_exe(line_nr, recdict, arg, 0)

def aap_child_and_exe(line_nr, recdict, arg, child):
    """Handle reading and possibly executing a recipe.
       "child" is non-zero for ":child", zero for ":execute"."""
    rpstack = getrpstack(recdict, line_nr)
    work = getwork(recdict)
    if child:
        cmdname = ":child"
    else:
        cmdname = ":execute"

    # When still starting up we can't create a child scope.
    if not recdict.get("_start"):
        recipe_error(rpstack,
                          _("%s cannot be used in a startup recipe") % cmdname)

    # Evaluate the options and arguments
    optiondict, attrdict, varlist = get_args(line_nr, recdict, arg, 
        {"p": "pass", "pass" : "pass",
         "n": "nopass", "nopass" : "nopass"})
    if attrdict:
        option_error(rpstack, attrdict, cmdname)

    if len(varlist) == 0:
        recipe_error(rpstack, _("%s requires an argument") % cmdname)
    if child and len(varlist) > 1:
        recipe_error(rpstack, _("%s only accepts one argument") % cmdname)

    varlist = dictlist_expand(varlist)

    name = varlist[0]["name"]

    force_fetch = Global.cmd_args.has_option("fetch-recipe")
    if ((force_fetch or not os.path.exists(name))
            and varlist[0].has_key("fetch")):
        # Need to create a node to fetch it.
        # Ignore errors, a check for existence is below.
        # Use a cached file when no forced fetch.
        from VersCont import fetch_nodelist
        node = work.get_node(name, 0, varlist[0])
        fetch_nodelist(rpstack, recdict, [ node ], not force_fetch)

    if not os.path.exists(name):
        if varlist[0].has_key("fetch"):
            recipe_error(rpstack, _('Cannot download recipe "%s"') % name)
        else:
            recipe_error(rpstack, _('Recipe "%s" does not exist') % name)

    try:
        cwd = os.getcwd()
    except OSError:
        recipe_error(rpstack, _("Cannot obtain current directory"))

    # Get the directory name of the child relative to the parent before
    # changing directory.
    childdir = shorten_name(os.path.dirname(name), cwd)

    if not child:
        msg_extra(recdict, _('Executing recipe "%s"') % name)

    # Change to the directory where the recipe is located.
    name = recipe_dir(recdict, os.path.abspath(name))

    #
    # Create a new scope and/or Work object to execute the recipe with.
    # ":child"              inherit scope, no new Work
    # ":child {nopass}"     new toplevel scope, no new Work
    # ":execute {pass}"     inherit scope, new Work
    # ":execute"            new toplevel scope, new Work
    #
    from Scope import get_build_recdict, create_topscope

    # Create a new scope for the executed/child recipe.
    if ((child and not optiondict.get("nopass"))
            or (not child and optiondict.get("pass"))):
        # Pass on the variables available in the current scope.
        new_recdict = get_build_recdict(recdict, None, recipe_name = name)
    else:
        # Hide everything from the current scope, create a new toplevel scope.
        if child: 
            # ":child" command without passing the current scope: Create a new
            # toplevel scope and set the default values.
            new_recdict = create_topscope(name).data
            new_recdict["_default"] = recdict["_default"]
            new_recdict["_start"] = recdict["_start"]

            # Set the startup values from the "_start" scope.  Do not read the
            # startup recipes again, the dependencies and rules are already
            # defined.
            fromdict = recdict["_start"].data
            for k in fromdict.keys():
                if k[0] != '_':
                    new_recdict[k] = fromdict[k]

            # Use the current Work object.
            setwork(new_recdict, work)
        else:
            # ":execute": let Work() create a new toplevel scope below.
            new_recdict = None

    if child:
        newwork = work
    else:
        # ":execute": Create a new Work object.

        # Get the arguments like command line arguments
        oldargs = Global.cmd_args
        Global.cmd_args = doargs(map(lambda x: x["name"], varlist[1:]))

        # Keep global options, e.g., --nobuild.
        copy_global_options(oldargs, Global.cmd_args)

        # Create a new Work object to execute the recipe in.
        oldwork = getwork(recdict)
        newwork = Work(new_recdict)
        newwork.top_recipe = name

        # Remember the top directory.
        newwork.top_dir = os.getcwd()

        # Need to read the startup recipes to get the default dependencies and
        # rules.  But when passing on the scope we don't use the variable
        # values.  Tricky!!!
        if new_recdict:
            rd = create_topscope(name).data
            setwork(rd, newwork)
        else:
            rd = newwork.recdict
            new_recdict = rd
        set_defaults(rd)

        # Set $TARGETARG.
        new_recdict["TARGETARG"] = list2str(Global.cmd_args.targets)

    # Add values set in the command line arguments
    add_cmdline_settings(new_recdict)

    # Always set the parent, also when a new toplevel is created.
    absdir = os.path.dirname(os.path.abspath(name))
    new_recdict["_parent"] = recdict["_recipe"]
    new_recdict["CHILDDIR"] = childdir
    new_recdict["PARENTDIR"] = shorten_name(cwd, os.path.dirname(name) or None)
    new_recdict["TOPDIR"] = shorten_name(absdir, newwork.top_dir)
    new_recdict["BASEDIR"] = shorten_name(newwork.top_dir, absdir)


    #
    # Read the recipe
    #
    read_recipe(rpstack, new_recdict, name, not child or Global.at_toplevel)

    from DoAddDef import doadddef
    if not child:
        #
        # ":execute" the recipe right now.
        #
        from DoBuild import dobuild
        doadddef(newwork, newwork.recdict, 1)
        dobuild(newwork)

        # Restore the previous Work object and continue there.
        setwork(recdict, oldwork)
        Global.cmd_args = oldargs

    else:
        # For a child recipe add default dependencies.
        doadddef(work, new_recdict, 0)

    # go back to the previous current directory
    try:
        goto_dir(recdict, cwd)
    except OSError:
        recipe_error(rpstack, _('Cannot change to directory "%s"') % cwd)


def aap_attr(line_nr, recdict, arg):
    """Add attributes to nodes."""
    aap_attribute(line_nr, recdict, arg)


def aap_attribute(line_nr, recdict, arg):
    """Add attributes to nodes."""
    rpstack = getrpstack(recdict, line_nr)
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg)
    if not argdictlist:
        recipe_error(rpstack, _(":attr command requires a file argument"))

    # Make file names relative to the current recipe.
    dict_expand(attrdict)

    # Expand wildcards.
    argdictlist = dictlist_expand(argdictlist)

    # Loop over all items, adding attributes to the node.
    work = getwork(recdict)
    for adict in argdictlist:
        if adict["name"] == '':
            from Dictlist import dictlistattr2str
            recipe_error(rpstack, _('Attribute without a name: %s') % dictlistattr2str(adict))

        node = work.get_node(adict["name"], 1, adict)
        node.set_attributes(attrdict)


def sep_scope(rpstack, name):
    """Separate the scope and the variable name out of "name".
       Returns (scope, name).  Scope is None when there is none."""
    i = string.find(name, ".")
    if i < 0:
        return None, name

    scope = name[:i]
    varname = name[i + 1:]
    if i == 0 or string.find(varname, ".") >= 0:
        recipe_error(rpstack, _("Invalid dot in variable name %s") % name)

    return scope, varname


def get_scope_recdict(rpstack, recdict, name, assign = 0):
    """
    Get the recdict, scope and varname to use for "name", which may include a
    scope name.  The returned scope is "_no" when no scope was specified.
    When a variable exists with the name of the scope the returned "rd" is the
    variable value.
    When "assign" is non-zero, create a new user scope when needed.
    """
    scope, varname = sep_scope(rpstack, name)
    if not scope:
        scope = "_no"
    try:
        rd = recdict[scope]
    except:
        from Scope import check_user_scope, create_user_scope
        if assign and scope[0] in string.letters:
            assert_scope_name(rpstack, scope)
            msg = check_user_scope(recdict, scope)
            if msg:
                recipe_error(rpstack, msg)
            create_user_scope(recdict, scope)
            rd = recdict[scope]
        else:
            recipe_error(rpstack, _("Invalid scope name: %s") % name)
    return rd, scope, varname


def aap_assign(line_nr, recdict, name, arg, dollar, extra):
    """Assignment command in a recipe.
       "name" is the name of the variable.
       "arg" is the argument value (Python expression already expanded).
       When "dollar" is '$' don't expand $VAR items.
       When "extra" is '?' only assign when "name" wasn't set yet.
       When "extra" is '+' append to "name"."""
    rpstack = getrpstack(recdict, line_nr)

    # Separate scope and decide which dictionary to use.
    rd, scope, varname = get_scope_recdict(rpstack, recdict, name, 1)

    # Skip the whole assignment for "var ?= val" if var was already set.
    if extra != '?' or not rd.has_key(varname):
        if dollar != '$':
            # Expand variables in "arg".
            val = expand(line_nr, recdict, arg, Expand(1, Expand.quote_aap))
        else:
            val = arg

        import types
        if rd.has_key(varname):
            v = rd.get(varname)
            from Scope import RecipeDict
            if isinstance(v, types.DictType) or isinstance(v, RecipeDict):
                if extra == '+':
                    # Appending to a scope is impossible, this is an error.
                    recipe_error(rpstack,
                               _("Cannot append to a scope: %s") % name)
                else:
                    # Overwriting a scope with a variable.  Is that an error?
                    msg_warning(recdict, _("Warning: Variable name already in use as a scope: %s") % name, rpstack = rpstack)

            if extra == '+':
                # append to old value
                oldval = get_var_val(line_nr, recdict, scope, varname)
                if oldval:
                    val = oldval + ' ' + val

        if dollar == '$':
            # Postpone expanding variables in "arg".  Create an ExpandVar
            # object  to remember it has to be done when using the variable.
            val = ExpandVar(val)

        # set the value
        try:
            rd[varname] = val
        except WriteAfterRead, e:
            recipe_error(rpstack, e)
        except TypeError:
            if not isinstance(rd, types.DictType):
                recipe_error(rpstack,
                               _("scope name refers to a variable: %s") % name)
            recipe_error(rpstack, _("Invalid scope name in %s") % name)


def expand(line_nr, recdict, arg, argexpand, startquote = '', skip_errors = 0):
    """Evaluate $VAR, $(VAR) and ${VAR} in "arg", which is a string.
       $VAR is expanded and the resulting string is combined with what comes
       before and after $VAR.  text$VAR  ->  "textval1"  "textval2".
       "argexpand" is an Expand object that specifies the way $VAR is expanded.
       When "startquote" isn't empty, work like "arg" was preceded by it.
       When "skip_errors" is non-zero, leave items with errors unexpanded,
       never fail.
       """
    rpstack = getrpstack(recdict, line_nr)
    res = ''                    # resulting string so far
    inquote = startquote        # ' or " when inside quotes
    itemstart = 0               # index where a white separated item starts
    arg_len = len(arg)
    idx = 0
    while idx < arg_len:
        if arg[idx] == '$':
            sidx = idx
            idx = idx + 1
            if arg[idx] == '$':
                res = res + '$'                 # reduce $$ to a single $
                idx = idx + 1
            elif arg[idx] == '#':
                res = res + '#'                 # reduce $# to a single #
                idx = idx + 1
            elif arg[idx] == '(' and idx + 2 < arg_len and arg[idx + 2] == ')':
                res = res + arg[idx + 1]        # reduce $(x) to x
                idx = idx + 3
            else:
                # Remember what non-white text came before the $.
                before = res[itemstart:]

                exp = copy.copy(argexpand)      # make a copy so that we can
                                                # change it

                while arg[idx] in '?-+*/=\'"\\!':
                    if arg[idx] == '?':         # optional existence
                        exp.optional = 1
                    elif arg[idx] == '-':       # exclude attributes
                        exp.attr = 0
                    elif arg[idx] == '+':       # include attributes
                        exp.attr = 1
                    elif arg[idx] == '*':       # no rc-style expansion
                        exp.rcstyle = 1
                    elif arg[idx] == '/':       # change slash to backslash
                        exp.slash = 1
                    elif arg[idx] == '=':       # no quoting
                        exp.quote = Expand.quote_none
                    elif arg[idx] == "'":       # A-A-P quoting
                        exp.quote = Expand.quote_aap
                    elif arg[idx] == '"':       # double quotes
                        exp.quote = Expand.quote_double
                    elif arg[idx] == '\\':      # backslash quoting
                        exp.quote = Expand.quote_bs
                    elif arg[idx] == '!':       # shell quoting
                        exp.quote = Expand.quote_shell
                    else:
                        print "Ooops!"
                    idx = idx + 1

                # Check for $(VAR) and ${VAR}.
                if arg[idx] == '(' or arg[idx] == '{':
                    s = skip_white(arg, idx + 1)
                else:
                    s = idx

                # get the variable name
                e = s
                while e < arg_len and varchar(arg[e]):
                    e = e + 1
                # Exclude a trailing dot, so that ":print did $target." works.
                if e > s and arg[e - 1] == '.':
                    e = e - 1
                if e == s:
                    if skip_errors:
                        res = res + arg[sidx:idx]
                        continue
                    recipe_error(rpstack, _("Invalid character after $"))

                name = arg[s:e]
                scope, varname = sep_scope(rpstack, name)
                if not scope:
                    # No scope specified, use "_no".
                    scope = "_no"
                try:
                    rd = recdict[scope]
                    scope_found = 1
                except:
                    scope_found = 0

                if scope_found:
                    try:
                        var_found = rd[varname]
                        var_found = 1
                    except:
                        var_found = 0

                if not (scope_found and var_found) and not exp.optional:
                    if skip_errors:
                        res = res + arg[sidx:idx]
                        continue
                    if not scope_found:
                        recipe_error(rpstack, _("Invalid scope name in %s") % name)
                    recipe_error(rpstack, _('Unknown variable: "%s"') % name)

                index = -1
                if s > idx:
                    # Inside () or {}
                    e = skip_white(arg, e)
                    if e < arg_len and arg[e] == '[':
                        # get index for $(var[n])
                        b = e
                        brak = 0
                        e = e + 1
                        # TODO: ignore [] inside quotes?
                        while e < arg_len and (arg[e] != ']' or brak > 0):
                            if arg[e] == '[':
                                brak = brak + 1
                            elif arg[e] == ']':
                                brak = brak - 1
                            e = e + 1
                        if e == arg_len or arg[e] != ']':
                            if skip_errors:
                                res = res + arg[sidx:idx]
                                continue
                            recipe_error(rpstack, _("Missing ']'"))
                        v = expand(line_nr, recdict, arg[b+1:e],
                                 Expand(0, Expand.quote_none), '', skip_errors)
                        try:
                            index = int(v)
                        except:
                            if skip_errors:
                                res = res + arg[sidx:idx]
                                continue
                            recipe_error(rpstack,
                                  _('index does not evaluate to a number: "%s"')
                                                                           % v)
                        if index < 0:
                            if skip_errors:
                                res = res + arg[sidx:idx]
                                continue
                            recipe_error(rpstack,
                                _('index evaluates to a negative number: "%d"')
                                                                       % index)
                        e = skip_white(arg, e + 1)

                    # Check for matching () and {}
                    if (e == arg_len
                            or (arg[idx] == '(' and arg[e] != ')')
                            or (arg[idx] == '{' and arg[e] != '}')):
                        if skip_errors:
                            res = res + arg[sidx:idx]
                            continue
                        recipe_error(rpstack, _('No match for "%s"') % arg[idx])

                    # Continue after the () or {}
                    idx = e + 1
                else:
                    # Continue after the varname
                    idx = e

                # Skip over optional variable that is not defined.
                if not (scope_found and var_found) and not exp.rcstyle:
                    continue

                # Find what comes after $VAR.
                # Need to remember whether it starts inside quotes.
                after_inquote = inquote
                s = idx
                while idx < arg_len:
                    if inquote:
                        if arg[idx] == inquote:
                            inquote = ''
                    elif arg[idx] == '"' or arg[idx] == "'":
                        inquote = arg[idx]
                    elif string.find(string.whitespace + "{", arg[idx]) != -1:
                        break
                    idx = idx + 1
                after = arg[s:idx]

                if exp.attr:
                    # Obtain any following attributes, advance to after them.
                    # Also expand $VAR inside the attributes.
                    attrdict, idx = get_attrdict(rpstack, recdict, arg, idx, 1)
                else:
                    attrdict = {}

                if exp.rcstyle and not (scope_found and var_found):
                    # xxx$?var results in nothing when $var doesn't exists
                    res = res[0:itemstart]
                    continue

                if not exp.rcstyle or (before == '' and after == ''
                                                       and len(attrdict) == 0):
                    if index < 0:
                        # No rc-style expansion or index, use the value of
                        # $VAR as specified with quote-expansion
                        try:
                            res = res + get_var_val(line_nr, recdict,
                                                           scope, varname, exp)
                        except (TypeError, KeyError):
                            # Get a KeyError for using a user scope as variable
                            # name.
                            if skip_errors:
                                res = res + arg[sidx:idx]
                                continue
                            from Scope import is_scope_name
                            if is_scope_name(recdict, varname):
                                recipe_error(rpstack,
                                        _('Using scope name as variable: "%s"')
                                                                     % varname)
                            recipe_error(rpstack,
                                    _('Type of variable "%s" must be a string')
                                                                     % varname)
                    else:
                        # No rc-style expansion but does have an index.
                        # Get the Dictlist of the referred variable.
                        varlist = str2dictlist(rpstack,
                                 get_var_val(line_nr, recdict, scope, varname))
                        if len(varlist) < index + 1:
                            if skip_errors:
                                # Note changing $(source[1]) to $(source[2]).
                                res = res + arg[sidx:idx]
                            else:
                                msg_warning(recdict,
                                    _('using %s[%d] but length is %d')
                                                 % (name, index, len(varlist)))
                        else:
                            res = res + expand_item(varlist[index], exp)
                    # TODO: Why was this here?
                    #for k in attrdict.keys():
                        #res = res + "{%s = %s}" % (k, attrdict[k])
                    # Continue with what comes after $VAR.
                    inquote = after_inquote
                    idx = s

                else:
                    # rc-style expansion of a variable

                    # Get the Dictlist of the referred variable.
                    # When an index is specified use that entry of the list.
                    # When index is out of range or the list is empty, use a
                    # list with one empty entry.
                    varlist1 = str2dictlist(rpstack,
                                 get_var_val(line_nr, recdict, scope, varname))
                    if (len(varlist1) == 0
                                or (index >= 0 and len(varlist1) < index + 1)):
                        if index >= 0 and not skip_errors:
                            msg_warning(recdict,
                              _('Index "%d" is out of range for variable "%s"')
                                                               % (index, name))
                        varlist1 = [{"name": ""}]
                    elif index >= 0:
                        varlist1 = [ varlist1[index] ]

                    # Evaluate the "after" of $(VAR)after {attr = val}.
                    varlist2 = str2dictlist(rpstack,
                                       expand(line_nr, recdict, after,
                                                   Expand(1, Expand.quote_aap),
                                                   startquote = after_inquote),
                                                   startquote = after_inquote)
                    if len(varlist2) == 0:
                        varlist2 = [{"name": ""}]

                    # If the referred variable is empty and "after" has only
                    # one item, the result is empty.  Thus "-L$*LIBS" is
                    # nothing when $LIBS is empty.
                    if (len(varlist1) == 1 and varlist1[0]["name"] == ""
                                                       and len(varlist2) == 1):
                        res = res[0:itemstart]
                    else:
                        # Remove quotes from "before", they are put back when
                        # needed.
                        lead = ''
                        q = ''
                        for c in before:
                            if q:
                                if c == q:
                                    q = ''
                                else:
                                    lead = lead + c
                            elif c == '"' or c == "'":
                                q = c
                            else:
                                lead = lead + c

                        # Combine "before", the list from $VAR, the list from
                        # "after" and the following attributes.
                        # Put "startquote" in front, because the terminalting
                        # quote will have been removed.
                        rcs = startquote
                        startquote = ''
                        for d1 in varlist1:
                            for d2 in varlist2:
                                if rcs:
                                    rcs = rcs + ' '
                                s = lead + d1["name"] + d2["name"]
                                # If $VAR was in quotes put the result in
                                # quotes.
                                if after_inquote:
                                    rcs = rcs + enquote(s,
                                                         quote = after_inquote)
                                else:
                                    rcs = rcs + expand_itemstr(s, exp)
                                if exp.attr:
                                    for k in d1.keys():
                                        if k != "name":
                                            rcs = rcs + "{%s = %s}" % (k, d1[k])
                                    for k in d2.keys():
                                        if k != "name":
                                            rcs = rcs + "{%s = %s}" % (k, d2[k])
                                    for k in attrdict.keys():
                                        rcs = rcs + "{%s = %s}" % (k, attrdict[k])
                        res = res[0:itemstart] + rcs

        else:
            # No '$' at this position, include the character in the result.
            # Check if quoting starts or ends and whether white space separates
            # an item, this is used for expanding $VAR.
            c = arg[idx]
            if inquote:
                if c == inquote:
                    inquote = ''
            elif c == '"' or c == "'":
                inquote = c
            elif c == ' ' or c == '\t':
                itemstart = len(res) + 1
            res = res + c
            idx = idx + 1
    return res


def shell_cmd_has_force(rpstack, recdict, s):
    """Return non-zero when "s" starts with a "force" attribute."""
    attrdict, i = get_attrdict(rpstack, recdict, s, 0, 1)
    if attrdict.get("f") or attrdict.get("force"):
        return 1
    return 0


def aap_shell(line_nr, recdict, cmds, async = -1):
    """Execute shell commands from the recipe.
       "cmds" must end in a newline character.
       When "async" is positive work asynchronously.
       When "async" is negative (the default) work asynchronously when the
       $async variable is set."""
    # Skip when not actually building.
    if skip_commands():
        if cmds[-1] == '\n':
            s = cmds[:-1]
        else:
            s = cmds
        msg_skip(line_nr, recdict, 'shell commands "%s"' % s)
        return

    rpstack = getrpstack(recdict, line_nr)

    # Need to get the "force" argument here.  Leave the attributes in the
    # command, they will be used by logged_system() as well.
    forced = shell_cmd_has_force(rpstack, recdict, cmds)

    cmd = expand(line_nr, recdict, cmds, Expand(0, Expand.quote_shell))

    if recdict.has_key("target"):
        msg_extra(recdict, _('Shell commands for updating "%s":')
                                                           % recdict["target"])

    if async < 0:
        async = recdict.get("async")
    if async and os.name in [ "posix", "nt" ]:
        # Run the command asynchronously.
        async_system(rpstack, recdict, cmd)
        n = 0
    else:
        n = logged_system(recdict, cmd)
    recdict["sysresult"] = n

    if n != 0 and not forced:
        recipe_error(getrpstack(recdict, line_nr),
                                            _("Shell command returned %d") % n)


def aap_system(line_nr, recdict, cmds):
    """Implementation of ":system cmds".  Almost the same as aap_shell()."""
    aap_shell(line_nr, recdict, cmds + '\n')


def aap_sys(line_nr, recdict, cmds):
    """Implementation of ":sys cmds".  Almost the same as aap_shell()."""
    aap_shell(line_nr, recdict, cmds + '\n')


def aap_sysdepend(line_nr, recdict, arg):
    """
    Implementation of ":sysdepend {filepat = pattern} shell-command".
    """
    rpstack = getrpstack(recdict, line_nr)
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    for k in attrdict.keys():
        if k != "filepat" and k != "srcpath":
            recipe_error(rpstack, _('Unknown option for :sysdepend: "%s"') % k)
    if not attrdict.get("filepat"):
        recipe_error(rpstack, _('filepat option missing in :sysdepend'))

    cmd = expand(line_nr, recdict, arg[i:], Expand(0, Expand.quote_shell))
    from RecPython import redir_system

    if attrdict.has_key("srcpath"):
        # Use the specified search path.
        searchpath = attrdict.get("srcpath")
    else:
        # Use the value of $INCLUDE, removing "-I".
        # Also look in the current directory.
        i = get_var_val(line_nr, recdict, "_no", "INCLUDE")
        if not i:
            searchpath = '.'
        else:
            searchpath = re.sub('^-I|[ "]-I', " ", i) + " ."

        # Also look in the directory of the source file, because most C
        # compilers will do this (e.g., compiling "test/foo.c" which contains
        # '#include "foo.h"' finds "test/foo.h").
        s = get_var_val(line_nr, recdict, "_no", "source")
        if s:
            s = str2dictlist(rpstack, s)[0]["name"]
            d = os.path.dirname(s)
            if d:
                searchpath = listitem2str(d) + ' ' + searchpath

    prev_files = []
    while 1:
        ok, text = redir_system(cmd)
        if ok:
            break

        # If files are missing try to fetch them and try again.
        files = []
        for line in string.split(text, '\n'):
            try:
                m = re.match(attrdict["filepat"], line)
            except StandardError, e:
                recipe_error(rpstack, _('error in filepat: %s') % str(e))
            if m:
                try:
                    f = m.group(1)
                except StandardError, e:
                    recipe_error(rpstack,
                        _('error using first group from filepat: %s') % str(e))
                # Trim white space from the file name.
                s = skip_white(f, 0)
                e = len(f)
                while e > s and is_white(f[e - 1]):
                    e = e - 1
                files.append(f[s:e])
        if not files:
            recipe_error(rpstack, _("Generating dependencies failed"))
        files_str = list2str(files)
        if files == prev_files:
            if len(files) == 1:
                msg_info(recdict, _("Cannot find included file: %s")
                                                                   % files_str)
            else:
                msg_info(recdict, _("Cannot find included files: %s")
                                                                   % files_str)
            break

        # Attempt fetching and/or updating the missing files.
        for afile in files:
            if searchpath:
                opt = "{searchpath = %s} " % searchpath
            else:
                opt = ''
            aap_update(line_nr, recdict, opt + listitem2str(afile))

        prev_files = files


def aap_syspath(line_nr, recdict, arg):
    """Implementation of ":syspath path arg"."""
    # Skip when not actually building.
    if skip_commands():
        msg_skip(line_nr, recdict, ':syspath ' + arg)
        return

    # Get the arguments into a dictlist
    rpstack = getrpstack(recdict, line_nr)
    xp = Expand(0, Expand.quote_shell)

    # Evaluate the arguments
    args = str2dictlist(rpstack,
                  expand(line_nr, recdict, arg, Expand(0, Expand.quote_aap)))
    if len(args) < 2:
        recipe_error(rpstack, _(":syspath requires at least two arguments"))

    path = args[0]["name"]
    path_len = len(path)
    i = 0
    while i < path_len:
        # Isolate one part of the path, until a colon, replacing %s with the
        # arguments, %% with % and %: with :.
        cmd = ''
        had_ps = 0
        while i < path_len and path[i] != ':':
            if path[i] == '%' and i + 1 < path_len:
                i = i + 1
                if path[i] == 's':
                    cmd = cmd + dictlist2str(args[1:], xp)
                    had_ps = 1
                else:
                    cmd = cmd + path[i]
            else:
                cmd = cmd + path[i]
            i = i + 1
        if not had_ps:
            cmd = cmd + ' ' + dictlist2str(args[1:], xp)

        if recdict.get("async") and os.name in [ "posix", "nt" ]:
            # Run the command asynchronously.
            # TODO: first check if the command exists.
            async_system(rpstack, recdict, cmd)
            return

        msg_system(recdict, cmd)
        if os.system(cmd) == 0:
            return
        i = i + 1

    recipe_error(rpstack, _('No working command found for :syspath "%s"')
                                                                        % path)


def aap_start(line_nr, recdict, cmds):
    """Implementation of ":start cmd"."""
    aap_shell(line_nr, recdict, cmds + '\n', async = 1)


def aap_copy(line_nr, recdict, arg):
    """Implementation of ":copy -x from to"."""
    # It's in a separate module, it's quite a bit of stuff.
    from CopyMove import copy_move
    copy_move(line_nr, recdict, arg, 1)


def aap_move(line_nr, recdict, arg):
    """Implementation of ":move -x from to"."""
    # It's in a separate module, it's quite a bit of stuff.
    from CopyMove import copy_move
    copy_move(line_nr, recdict, arg, 0)


def aap_symlink(line_nr, recdict, arg):
    """Implementation of ":symlink {f}{q} from to"."""
    # Skip when not actually building.
    if skip_commands():
        msg_skip(line_nr, recdict, ':symlink ' + arg)
        return
    rpstack = getrpstack(recdict, line_nr)

    # Get the arguments and check the options.
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"f": "force", "force" : "force",
             "q": "quiet", "quiet" : "quiet"})
    if attrdict:
        option_error(rpstack, attrdict, ":symlink")

    # Get the remaining arguments, should be at least one.
    if len(argdictlist) != 2:
        recipe_error(rpstack, _(":symlink command requires two arguments"))
    fromarg = argdictlist[0]["name"]
    toarg = argdictlist[1]["name"]

    if os.path.islink(toarg) or os.path.exists(toarg):
        if optiondict.get("force"):
            os.unlink(toarg)
        else:
            msg = _(':symlink: target exists: "%s"') % toarg
            if optiondict.get("quiet"):
                msg_extra(recdict, msg)
            else:
                recipe_error(rpstack, msg)
            return
    if not os.path.exists(fromarg) and not optiondict.get("quiet"):
        msg_warning(recdict, _('file linked to does not exist: "%s"')
                                                                     % fromarg)
    try:
        os.symlink(fromarg, toarg)
    except StandardError, e:
        msg = _(':symlink "%s" "%s" failed: %s') % (fromarg, toarg, str(e))
        if optiondict.get("quiet"):
            msg_extra(recdict, msg)
        else:
            recipe_error(rpstack, msg)


def aap_delete(line_nr, recdict, arg):
    """Alias for aap_del()."""
    aap_del(line_nr, recdict, arg)


def aap_del(line_nr, recdict, arg):
    """Implementation of ":del {r} file1 file2"."""
    # Skip when not actually building.
    if skip_commands():
        msg_skip(line_nr, recdict, ":delete " + arg)
        return
    rpstack = getrpstack(recdict, line_nr)
    work = getwork(recdict)

    # Get the arguments and check the options.
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"f": "force", "force" : "force",
             "q": "quiet", "quiet" : "quiet",
             "c" : "continue", "continue" : "continue",
             "r": "recursive", "recursive" : "recursive"},
             exp_attr = 0)
    if attrdict:
        option_error(rpstack, attrdict, ":delete")

    # Get the remaining arguments, should be at least one.
    if not argdictlist:
        recipe_error(rpstack, _(":delete command requires a file argument"))

    from Remote import url_split3

    for a in argdictlist:
        fname = a["name"]
        scheme, mach, path = url_split3(fname)
        if scheme != '':
            recipe_error(rpstack, _('Cannot delete remotely yet: "%s"') % fname)

        # Expand ~user and wildcards.
        fname = os.path.expanduser(fname)
        fl = glob.glob(fname)
        if len(fl) == 0:
            # glob() doesn't include a symbolic link if its destination doesn't
            # exist, we want to delete it anyway.
            if isalink(fname):
                fl = [ fname ]
            elif optiondict.get("force"):
                # No match, try without expanding.
                fl = [ fname ]
            elif optiondict.get("continue") and has_wildcard(fname):
                # Skip
                msg_note(recdict, _("No match for %s") % fname)
                continue
            else:
                recipe_error(rpstack, _('No such file or directory: "%s"')
                                                                       % fname)

        for f in fl:
            f_msg = shorten_name(f, work.top_dir)
            isdir = os.path.isdir(f)
            islink = isalink(f)
            try:
                if optiondict.get("recursive"):
                    deltree(f)
                else:
                    os.remove(f)
            except EnvironmentError, e:
                msg = (_('Could not delete "%s"') % f_msg) + str(e)
                if optiondict.get("force"):
                    msg_note(recdict, msg)
                    continue
                recipe_error(rpstack, msg)
            else:
                if os.path.exists(f):
                    msg = _('Could not delete "%s"') % f_msg
                    if optiondict.get("force"):
                        msg_note(recdict, msg)
                        continue
                    recipe_error(rpstack, msg)
            if not optiondict.get("quiet"):
                if islink:
                    msg_info(recdict, _('Deleted link "%s"') % f_msg)
                elif isdir:
                    msg_info(recdict, _('Deleted directory tree "%s"') % f_msg)
                else:
                    msg_info(recdict, _('Deleted "%s"') % f_msg)


def aap_deldir(line_nr, recdict, arg):
    """Implementation of ":deldir dir1 dir2"."""
    # Skip when not actually building.
    if skip_commands():
        msg_skip(line_nr, recdict, ':deldir ' + arg)
        return
    rpstack = getrpstack(recdict, line_nr)

    # Get the arguments and check the options.
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"f": "force", "force" : "force",
             "q": "quiet", "quiet" : "quiet"})
    if attrdict:
        option_error(rpstack, attrdict, ":deldir")

    # Get the remaining arguments, should be at least one.
    if not argdictlist:
        recipe_error(rpstack,
                            _(":deldir command requires a directory argument"))

    from Remote import url_split3

    # Loop over the arguments
    for a in argdictlist:
        item = a["name"]
        scheme, mach, path = url_split3(item)
        if scheme != '':
            recipe_error(rpstack, _('Cannot delete remotely yet: "%s"') % item)

        # Expand ~user and wildcards.
        dirlist = glob.glob(os.path.expanduser(item))
        if len(dirlist) == 0 and not optiondict.get("force"):
            recipe_error(rpstack, _('No match for "%s"') % item)
        
        # Loop over expanded items.
        for adir in dirlist:
            if not os.path.exists(adir):
                if not optiondict.get("force"):
                    recipe_error(rpstack, _('"%s" does not exists') % adir)
            elif not os.path.isdir(adir):
                recipe_error(rpstack, _('"Not a directory: "%s"') % adir)
            else:
                try:
                    os.rmdir(adir)
                except StandardError, e:
                    if os.path.exists(adir):
                        recipe_error(rpstack, (_('Could not delete "%s"') % adir)
                                                                      + str(e))
                else:
                    if not optiondict.get("quiet"):
                        msg_info(recdict, _('Deleted directory "%s"') % adir)


def aap_mkdir(line_nr, recdict, arg):
    """Implementation of ":mkdir dir1 dir2"."""
    # Skip when not actually building.
    if skip_commands():
        msg_skip(line_nr, recdict, ':mkdir ' + arg)
        return
    rpstack = getrpstack(recdict, line_nr)

    # Get the arguments and check the options.
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"f": "force", "force" : "force",
             "q": "quiet", "quiet" : "quiet",
             "r": "recursive", "recursive" : "recursive"})
    if attrdict:
        option_error(rpstack, attrdict, ":mkdir")
    if not argdictlist:
        recipe_error(rpstack, _(":mkdir command requires an argument"))

    from Remote import url_split3

    for a in argdictlist:
        name = a["name"]
        scheme, mach, path = url_split3(name)
        if scheme != '':
            recipe_error(rpstack, _('Cannot create remote directory yet: "%s"')
                                                                       % name)
        # Expand ~user, create directory
        adir = os.path.expanduser(name)

        # Skip creation when it already exists.
        if os.path.exists(adir):
            if not os.path.isdir(adir):
                recipe_error(rpstack, _('"%s" exists but is not a directory')
                                                                         % adir)
            if not optiondict.get("force"):
                recipe_error(rpstack, _('"%s" already exists') % adir)
        else:
            try:
                if optiondict.get("recursive"):
                    if a.get("mode"):
                        os.makedirs(adir, oct2int(a["mode"]))
                    else:
                        os.makedirs(adir)
                else:
                    if a.get("mode"):
                        os.mkdir(adir, oct2int(a["mode"]))
                    else:
                        os.mkdir(adir)
            except EnvironmentError, e:
                recipe_error(rpstack, (_('Could not create directory "%s"')
                                                               % adir) + str(e))
            else:
                if not optiondict.get("quiet"):
                    msg_info(recdict, _('Created directory "%s"') % adir)


def aap_changed(line_nr, recdict, arg):
    """Implementation of ":changed File ...".
       "line_nr" is used for error messages."""
    rpstack = getrpstack(recdict, line_nr)

    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"r": "recursive", "recursive" : "recursive"})
    if attrdict:
        option_error(rpstack, attrdict, ":changed")
    if not argdictlist:
        recipe_error(rpstack, _(":changed command requires a file argument"))

    from Sign import sign_clear_file
    for a in argdictlist:
        sign_clear_file(a["name"], optiondict.get("recursive"))


def aap_touch(line_nr, recdict, arg):
    """Implementation of ":touch file1 file2"."""
    # Skip when not actually building.
    if skip_commands():
        msg_skip(line_nr, recdict, ':touch ' + arg)
        return
    rpstack = getrpstack(recdict, line_nr)

    # Get the arguments and check the options.
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"f": "force", "force" : "force",
             "e": "exist", "exist" : "exist", "exists": "exist"})
    if attrdict:
        option_error(rpstack, attrdict, ":touch")

    # Get the remaining arguments, should be at least one.
    if not argdictlist:
        recipe_error(rpstack, _(":touch command requires a file argument"))

    from Remote import url_split3
    import time

    for a in argdictlist:
        name = a["name"]
        scheme, mach, path = url_split3(name)
        if scheme != '':
            recipe_error(rpstack, _('Cannot touch remote file yet: "%s"')
                                                                       % name)
        # Expand ~user, touch file
        name = os.path.expanduser(name)
        if os.path.exists(name):
            if optiondict.get("exist"):
                continue
            now = time.time()
            try:
                os.utime(name, (now, now))
            except EnvironmentError, e:
                recipe_error(rpstack, (_('Could not update time of "%s"')
                                                              % name) + str(e))
        else:
            if not optiondict.get("force") and not optiondict.get("exist"):
                recipe_error(rpstack,
                     _('"%s" does not exist (use :touch {force} to create it)')
                                                                        % name)
            try:
                # create an empty file or directory
                if a.get("directory"):
                    if a.get("mode"):
                        os.makedirs(name, oct2int(a["mode"]))
                    else:
                        os.makedirs(name)
                else:
                    if a.get("mode"):
                        touch_file(name, oct2int(a["mode"]))
                    else:
                        touch_file(name, 0644)
            except EnvironmentError, e:
                recipe_error(rpstack, (_('Could not create "%s"')
                                                              % name) + str(e))

def touch_file(name, mode):
    """Unconditionally create empty file "name" with mode "mode"."""
    f = os.open(name, os.O_WRONLY + os.O_CREAT + os.O_EXCL, mode)
    os.close(f)


def aap_chmod(line_nr, recdict, arg):
    """Implementation of ":chmod mode file1 file2"."""
    # Skip when not actually building.
    if skip_commands():
        msg_skip(line_nr, recdict, ':chmod ' + arg)
        return
    rpstack = getrpstack(recdict, line_nr)

    # Get the arguments and check the options.
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"f" : "force", "force" : "force",
             "c" : "continue", "continue" : "continue"})
    if attrdict:
        option_error(rpstack, attrdict, ":chmod")

    # Get the remaining arguments, should be at least one.
    if len(argdictlist) < 2:
        recipe_error(rpstack, _(":chmod command requires two arguments"))

    try:
        mode = oct2int(argdictlist[0]["name"])
    except UserError, e:
        recipe_error(rpstack, _("in :chmod command: ") + str(e))

    from Remote import url_split3

    # Loop over the file name arguments.
    for a in argdictlist[1:]:
        item = a["name"]
        scheme, mach, path = url_split3(item)
        if scheme != '':
            recipe_error(rpstack, _('Cannot chmod a remote file yet: "%s"')
                                                                       % item)
        # Expand ~user and wildcards.
        fname = os.path.expanduser(item)
        filelist = glob.glob(fname)
        if len(filelist) == 0:
            # No matching files.
            if optiondict.get("force"):
                # Try without expanding
                filelist = [ fname ]
            elif optiondict.get("continue") and has_wildcard(fname):
                # Skip
                continue
            else:
                recipe_error(rpstack, _('No match for "%s"') % item)
        
        # Loop over expanded items.
        for fname in filelist:
            if not os.path.exists(fname):
                if not optiondict.get("force"):
                    recipe_error(rpstack, _('"%s" does not exists') % fname)
            else:
                try:
                    os.chmod(fname, mode)
                except StandardError, e:
                    recipe_error(rpstack, (_('Could not chmod "%s"') % fname)
                                                                      + str(e))


# dictionary of recipes that have been fetched (using full path name).
recipe_fetched = {}


def aap_include(line_nr, recdict, arg):
    """Handle ":include filename": read the recipe into the current recdict."""
    work = getwork(recdict)
    rpstack = getrpstack(recdict, line_nr)

    # Evaluate the options and arguments
    optiondict, attrdict, args = get_args(line_nr, recdict, arg,
            {"q": "quiet", "quiet" : "quiet",
             "o": "once", "once" : "once"})
    if attrdict:
        option_error(rpstack, attrdict, ":include")

    if len(args) != 1:
        recipe_error(rpstack, _(":include requires one argument"))
    args = dictlist_expand(args)
    recname = args[0]["name"]

    includelist = Global.cmd_args.options.get("include")
    if not os.path.isabs(recname) and recname[0] != '.' and includelist:
        # Search for the recipe in the list of include directories.
        for adir in [ "." ] + includelist:
            n = os.path.join(adir, recname)
            if os.path.isfile(n):
                recname = n
                break

    if optiondict.get("once") and did_read_recipe(work, recname):
        msg_extra(recdict, _('Skipping recipe already read: %s"') % recname)
        return

    # Fetch the recipe when invoked with the "-R" argument.
    if ((Global.cmd_args.has_option("fetch-recipe")
            or not os.path.exists(recname))
                and args[0].has_key("fetch")):
        # Use the original recipe name, without the directory of "-I dir" added.
        fetchname = args[0]["name"]
        fullname = full_fname(fetchname)
        if not recipe_fetched.has_key(fullname):
            from VersCont import fetch_nodelist

            # Create a node for the recipe and fetch it.
            node = work.get_node(fetchname, 0, args[0])
            if fetch_nodelist(rpstack, recdict, [ node ], 0):
                msg_warning(recdict,
                                 _('Could not update recipe "%s"') % fetchname)
            else:
                recname = fetchname

            # Also mark it as updated when it failed, don't try again.
            recipe_fetched[fullname] = 1

    read_recipe(rpstack, recdict, recname, Global.at_toplevel,
                                            optional = optiondict.get("quiet"))


def aap_import(line_nr, recdict, arg):
    """
    Handle :import commands. Also used automagically for :produce
    targets, which specify a new language.
    """
    # Boilerplate - find the work object and stack for the current recipe.
    work = getwork(recdict)
    rpstack = getrpstack(recdict, line_nr)

    # Evaluate the options and arguments - there are no options, so complain if
    # any are given.
    optiondict, attrdict, args = get_args(line_nr, recdict, arg, { } )
    if attrdict:
        option_error(rpstack, attrdict, ":import")
    if optiondict:
        option_error(rpstack, optiondict, ":import")

    if len(args) != 1:
        recipe_error(rpstack, _(":import requires one argument"))
    name = args[0]["name"]

    # Import only imports a given module once.
    if work.module_already_read.has_key(name):
        msg_extra(recdict, _('Skipping module already imported: %s"') % name)
        return
    work.module_already_read[name] = 1

    # Check if the module name can be used for a scope name.
    from Scope import check_user_scope
    scope_name = "m_" + name
    assert_scope_name(rpstack, scope_name)
    rd = recdict.get(scope_name)
    if rd:
        from UserDict import UserDict
        if isinstance(rd, UserDict):
            recipe_error(rpstack, _('module name conflicts existing scope: %s')
                                                                  % scope_name)
        else:
            recipe_error(rpstack, _('module name conflicts with variable: %s')
                                                                  % scope_name)
    msg = check_user_scope(recdict, scope_name)
    if msg:
        recipe_error(rpstack, msg)

    # Like a ":child" command, but only passing the _top scope, not the recipe
    # tree: Create a new scope.
    from Scope import get_build_recdict
    new_recdict = get_build_recdict(recdict["_top"], None, recipe_name = name)

    # Add the module name as a scope name.  Use a RecipeDict to make
    # "m_name.var" work in Python commands.
    from Scope import add_user_scope, RecipeDict
    scope_recdict = RecipeDict()
    scope_recdict.data = new_recdict
    add_user_scope(new_recdict, scope_name, scope_recdict)

    # Use the current Work object.
    setwork(new_recdict, work)

    dirs = map(lambda x: os.path.join(x, "modules"), get_import_dirs(recdict))
    msg_extra(recdict, _('Importing "%s" from %s') % (name, str(dirs)))
    done = 0
    for adir in dirs:
        # Read the imported recipe if it exists in "adir".  This will produce an
        # error if the recipe exists but cannot be read.
        recname = os.path.join(adir, name) + ".aap"
        if os.path.exists(recname):
            read_recipe(rpstack, new_recdict, recname, 1)
            done = 1
            break

    if not done:
        # TODO: download the module (if the user wants this)
        recipe_error(rpstack, _('Cannot import "%s"') % name)

    # Also read the extra settings from "modules2/".
    for d in default_dirs(recdict, homedirfirst = 1):
        recname = os.path.join(os.path.join(d, "modules2"), name) + ".aap"
        if os.path.exists(recname):
            read_recipe(rpstack, new_recdict, recname, 1)


def aap_toolsearch(line_nr, recdict, arg):
    """
    :toolsearch tool1 tool2 ...
    """
    rpstack = getrpstack(recdict, line_nr)

    # Evaluate the options and arguments - there are no options, so complain if
    # any are given.
    optiondict, attrdict, args = get_args(line_nr, recdict, arg, { } )
    if attrdict:
        option_error(rpstack, attrdict, ":toolsearch")
    if optiondict:
        option_error(rpstack, optiondict, ":toolsearch")

    # only import "name" from tools directories
    tools_path = [ os.path.join(d, "tools") 
                   for d in get_import_dirs(recdict) ]
    didone = 0
    for a in args:
        c = a["name"]
        fpd = imp.find_module(c, tools_path)
        exec "tools_%s = imp.load_module(c, *fpd)" % c
        if eval("tools_%s.exists()" % c):
            exec "tools_%s.define_actions()" % c
            if not didone:
                didone = 1
                exec "tools_%s.use_actions(recdict['_top'])" % c


def maydo_recipe_cmd(rpstack):
    """Return non-zero if a ":recipe" command in the current recipe may be
    executed."""

    # Return right away when not invoked with the "-R" argument.
    if not Global.cmd_args.has_option("fetch-recipe"):
        return 0

    # Skip when this recipe was already updated.
    recname = full_fname(rpstack[-1].name)
    if recipe_fetched.has_key(recname):
        return 0

    return 1


def aap_recipe(line_nr, recdict, arg):
    """Handle ":recipe {fetch = name_list}": may download this recipe."""

    work = getwork(recdict)
    rpstack = getrpstack(recdict, line_nr)

    # Return right away when not to be executed.
    if not maydo_recipe_cmd(rpstack):
        return

    # Register the recipe to have been updated.  Also when it failed, don't
    # want to try again.
    recname = full_fname(rpstack[-1].name)
    recipe_fetched[recname] = 1

    short_name = shorten_name(recname)
    msg_info(recdict, _('Updating recipe "%s"') % short_name)

    orgdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    if not orgdict.has_key("fetch"):
        recipe_error(rpstack, _(":recipe requires a fetch attribute"))
    # TODO: warning for trailing characters?

    from VersCont import fetch_nodelist

    # Create a node for the recipe and fetch it.
    node = work.get_node(short_name, 0, orgdict)
    if not fetch_nodelist(rpstack, recdict, [ node ], 0):
        # TODO: check if the recipe was completely read
        # TODO: no need for restart if the recipe didn't change

        # Restore the recdict to the values from when starting to read the
        # recipe.
        start_recdict = recdict["_start_recdict"]
        for k in recdict.keys():
            if not start_recdict.has_key(k):
                del recdict[k]
        for k in start_recdict.keys():
            recdict[k] = start_recdict[k]

        # read the new recipe file
        read_recipe(rpstack, recdict, recname, Global.at_toplevel, reread = 1)

        # Throw an exception to cancel executing the rest of the script
        # generated from the old recipe.  This is catched in read_recipe()
        raise OriginUpdate

    msg_warning(recdict, _('Could not update recipe "%s"') % node.name)

#
# Generic function for getting the arguments of :fetch, :checkout, :commit,
# :checkin, :unlock and :publish
#
def get_verscont_args(line_nr, recdict, arg, cmd):
    """"Handle ":cmd {attr = } file ..."."""
    rpstack = getrpstack(recdict, line_nr)

    # Get the optional attributes that apply to all arguments.
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)

    # evaluate the arguments into a dictlist
    varlist = str2dictlist(rpstack,
              expand(line_nr, recdict, arg[i:], Expand(1, Expand.quote_aap)))
    if not varlist:
        recipe_error(rpstack, _(':%s requires an argument') % cmd)

    varlist = dictlist_expand(varlist)

    return attrdict, varlist


def do_verscont_cmd(rpstack, recdict, action, attrdict, varlist):
    """Perform "action" on items in "varlist", using attributes in
       "attrdict"."""
    from VersCont import verscont_nodelist, fetch_nodelist, publish_nodelist
    work = getwork(recdict)

    # Turn the dictlist into a nodelist.
    # Skip files that exist for "fetch" with the "constant" attribute.
    nodelist = []
    for item in varlist:
        node = work.get_node(item["name"], 1, item)
        node.set_attributes(attrdict)
        if action != "fetch" or node.may_fetch():
            nodelist.append(node)

    # Perform the action on the nodelist
    if nodelist:
        if action == "fetch":
            failed = fetch_nodelist(rpstack, recdict, nodelist, 0)
        elif action == "publish":
            failed = publish_nodelist(rpstack, recdict, nodelist, 1)
        else:
            failed = verscont_nodelist(rpstack, recdict, nodelist, action)
        if failed:
            recipe_error(rpstack, _('%s failed for "%s"') % (action,
                                   str(map(lambda x: x.short_name(), failed))))

def verscont_cmd(line_nr, recdict, arg, action):
    """Perform "action" for each item "varlist"."""
    rpstack = getrpstack(recdict, line_nr)
    attrdict, varlist = get_verscont_args(line_nr, recdict, arg, action)
    do_verscont_cmd(rpstack, recdict, action, attrdict, varlist)


def aap_fetch(line_nr, recdict, arg):
    """"Handle ":fetch {attr = val} file ..."."""
    verscont_cmd(line_nr, recdict, arg, "fetch")

def aap_checkout(line_nr, recdict, arg):
    """"Handle ":checkout {attr = val} file ..."."""
    verscont_cmd(line_nr, recdict, arg, "checkout")

def aap_commit(line_nr, recdict, arg):
    """"Handle ":commit {attr = val} file ..."."""
    verscont_cmd(line_nr, recdict, arg, "commit")

def aap_checkin(line_nr, recdict, arg):
    """"Handle ":checkin {attr = val} file ..."."""
    verscont_cmd(line_nr, recdict, arg, "checkin")

def aap_unlock(line_nr, recdict, arg):
    """"Handle ":unlock {attr = val} file ..."."""
    verscont_cmd(line_nr, recdict, arg, "unlock")

def aap_publish(line_nr, recdict, arg):
    """"Handle ":publish {attr = val} file ..."."""
    verscont_cmd(line_nr, recdict, arg, "publish")

def aap_add(line_nr, recdict, arg):
    """"Handle ":add {attr = val} file ..."."""
    verscont_cmd(line_nr, recdict, arg, "add")

def aap_remove(line_nr, recdict, arg):
    """"Handle ":remove {attr = val} file ..."."""
    verscont_cmd(line_nr, recdict, arg, "remove")

def aap_tag(line_nr, recdict, arg):
    """"Handle ":tag {attr = val} file ..."."""
    verscont_cmd(line_nr, recdict, arg, "tag")


def aap_verscont(line_nr, recdict, arg):
    """"Handle ":verscont action {attr = val} [file ...]"."""
    rpstack = getrpstack(recdict, line_nr)

    # evaluate the arguments into a dictlist
    varlist = str2dictlist(rpstack,
                  expand(line_nr, recdict, arg, Expand(1, Expand.quote_aap)))
    if not varlist:
        recipe_error(rpstack, _(':verscont requires an argument'))

    if len(varlist) > 1:
        arglist = dictlist_expand(varlist[1:])
    else:
        arglist = []
    do_verscont_cmd(rpstack, recdict, varlist[0]["name"], varlist[0], arglist)


def do_fetch_all(rpstack, recdict, attrdict):
    """Fetch all nodes with a "fetch" or "commit" attribute.
       Return non-zero for success."""
    work = getwork(recdict)

    from VersCont import fetch_nodelist

    nodelist = []
    for node in work.nodes.values():
        # Only need to fetch a node when:
        # - it has an "fetch" attribute
        # - the node doesn't exist yet
        # - it does exist and the "constant" attribute isn't set
        if ((node.attributes.has_key("fetch")
                    or node.attributes.has_key("commit"))
                and node.may_fetch()):
            node.set_attributes(attrdict)
            nodelist.append(node)

    if nodelist and fetch_nodelist(rpstack, recdict, nodelist, 0):
        ok = 0
    else:
        ok = 1

    return ok

def aap_fetchall(line_nr, recdict, arg):
    """"Handle ":fetchall {attr = val}"."""
    rpstack = getrpstack(recdict, line_nr)
    # Get the optional attributes that apply to all nodes.
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    do_fetch_all(rpstack, recdict, attrdict)


def do_verscont_all(rpstack, recdict, action, attrdict):
    """Do version control action "action" on all nodes with the "commit"
       attribute.
       Apply items from dictionary 'attrdict" to each node.
       Return non-zero for success."""
    work = getwork(recdict)

    from VersCont import verscont_nodelist

    # Loop over all nodes.
    nodelist = []
    for node in work.nodes.values():
        if (node.attributes.has_key("commit")
                and (action != "add" or node.attributes.has_key("tag"))):
            node.set_attributes(attrdict)
            nodelist.append(node)

    if nodelist and verscont_nodelist(rpstack, recdict, nodelist, action):
        ok = 0
    else:
        ok = 1

    return ok

def aap_commitall(line_nr, recdict, arg):
    """"Handle ":commitall {attr = val}"."""
    rpstack = getrpstack(recdict, line_nr)
    # Get the optional attributes that apply to all nodes.
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    do_verscont_all(rpstack, recdict, "commit", attrdict)


def aap_checkinall(line_nr, recdict, arg):
    """"Handle ":checkinall {attr = val}"."""
    rpstack = getrpstack(recdict, line_nr)
    # Get the optional attributes that apply to all nodes.
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    do_verscont_all(rpstack, recdict, "checkin", attrdict)


def aap_checkoutall(line_nr, recdict, arg):
    """"Handle ":checkoutall {attr = val}"."""
    rpstack = getrpstack(recdict, line_nr)
    # Get the optional attributes that apply to all nodes.
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    do_verscont_all(rpstack, recdict, "checkout", attrdict)


def aap_unlockall(line_nr, recdict, arg):
    """"Handle ":unlockall {attr = val}"."""
    rpstack = getrpstack(recdict, line_nr)
    # Get the optional attributes that apply to all nodes.
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    do_verscont_all(rpstack, recdict, "unlock", attrdict)


def aap_tagall(line_nr, recdict, arg):
    """"Handle ":tagall {attr = val}"."""
    rpstack = getrpstack(recdict, line_nr)
    # Get the optional attributes that apply to all nodes.
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    do_verscont_all(rpstack, recdict, "tag", attrdict)


def do_revise_all(rpstack, recdict, attrdict, local):
    res1 = do_verscont_all(rpstack, recdict, "checkin", attrdict)
    res2 = do_remove_add(rpstack, recdict, attrdict, local, "remove")
    return res1 and res2

def aap_reviseall(line_nr, recdict, arg):
    """"Handle ":reviseall {attr = val}"."""
    rpstack = getrpstack(recdict, line_nr)

    # Get the arguments and check the options.
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"l": "local", "local" : "local"})
    if argdictlist:
        recipe_error(rpstack, _('Too many arguments for :reviseall'))

    do_revise_all(rpstack, recdict, attrdict, optiondict.get("local"))


def do_publish_all(rpstack, recdict, attrdict):
    """Publish all noces with a "publish" attribute.
       Returns a list of nodes that failed."""
    work = getwork(recdict)

    from VersCont import publish_nodelist

    # Loop over all nodes.
    nodelist = []
    for node in work.nodes.values():
        if node.attributes.has_key("publish"):
            node.set_attributes(attrdict)
            nodelist.append(node)

    if nodelist:
        failed = publish_nodelist(rpstack, recdict, nodelist, 0)
    else:
        msg_extra(recdict, _('nothing to be published'))
        failed = []

    return failed

def aap_publishall(line_nr, recdict, arg):
    """"Handle ":publishall {attr = val}"."""
    rpstack = getrpstack(recdict, line_nr)
    # Get the optional attributes that apply to all nodes.
    attrdict, i = get_attrdict(rpstack, recdict, arg, 0, 1)
    failed = do_publish_all(rpstack, recdict, attrdict)
    if failed:
        recipe_error(rpstack, _('publish failed for "%s"')
                                % (str(map(lambda x: x.short_name(), failed))))


def do_remove_add(rpstack, recdict, attrdict, local, action):
    """When "action" is "remove": Remove all files from VCS that don't appear
       in the recipe or don't have the "commit" attribute.
       When "action" is "add": Add files to VCS that appear in the recipe with
       the "commit" attribute but don't appear in the VCS.
       Returns non-zero for success."""
    # Skip when not actually building.
    if skip_commands():
        msg_info(recdict, _('skip %sall') % action)
        return 1

    attrdict["name"] = "."
    assert_attribute(recdict, attrdict, "commit")

    from VersCont import verscont_remove_add
    return verscont_remove_add(rpstack, recdict, attrdict, not local, action)

def aap_remove_add(line_nr, recdict, arg, action):
    """Common code for ":removeall" and ":addall"."""
    # Skip when not actually building.
    if skip_commands():
        msg_skip(line_nr, recdict, ':%sall %s' % (action, arg))
        return
    rpstack = getrpstack(recdict, line_nr)

    # Get the arguments and check the options.
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
            {"l": "local", "local" : "local",
             "r": "recursive", "recursive" : "recursive"})

    from VersCont import verscont_remove_add

    if argdictlist:
        argdictlist = dictlist_expand(argdictlist)

        # Directory name arguments: Do each directory non-recursively
        for adir in argdictlist:
            for k in attrdict.keys():
                adir[k] = attrdict[k]
            assert_attribute(recdict, adir, "commit")
            verscont_remove_add(rpstack, recdict, adir,
                                           optiondict.get("recursive"), action)
    else:
        # No arguments: Do current directory recursively
        do_remove_add(rpstack, recdict, attrdict,
                                               optiondict.get("local"), action)

def aap_removeall(line_nr, recdict, arg):
    """"Handle ":removeall {attr = val} [dir ...]"."""
    aap_remove_add(line_nr, recdict, arg, "remove")

def aap_addall(line_nr, recdict, arg):
    """"Handle ":addall {attr = val}"."""
    aap_remove_add(line_nr, recdict, arg, "add")


def aap_filetype(line_nr, recdict, arg, cmd_line_nr, commands):
    """Add filetype detection from a file or in-line detection rules."""
    from Filetype import ft_add_rules, ft_read_file, DetectError
    rpstack = getrpstack(recdict, line_nr)

    # look through the arguments
    args = str2dictlist(rpstack,
                  expand(line_nr, recdict, arg, Expand(0, Expand.quote_aap)))
    if len(args) > 1:
        recipe_error(rpstack, _('Too many arguments for :filetype'))
    if len(args) == 1 and commands:
        recipe_error(rpstack,
                         _('Cannot have file name and commands for :filetype'))
    if len(args) == 0 and not commands:
        recipe_error(rpstack,
                            _('Must have file name or commands for :filetype'))

    try:
        if commands:
            what = "lines"
            ft_add_rules(commands, cmd_line_nr, recdict)
        else:
            fname = args[0]["name"]
            what = 'file "%s"' % fname
            ft_read_file(fname, recdict)
    except DetectError, e:
        recipe_error(rpstack, (_('Error in detection %s: ') % what) + str(e))


def aap_action(line_nr, recdict, arg, cmd_line_nr, commands):
    """Add an application for an action-filetype pair."""
    from Action import action_add

    rpstack = getrpstack(recdict, line_nr)
    action_add(rpdeepcopy(rpstack, cmd_line_nr),
            recdict,
            str2dictlist(rpstack,
                 expand(line_nr, recdict, arg, Expand(1, Expand.quote_aap))),
                                                                      commands)

def aap_usetool(line_nr, recdict, arg):
    """
    Set a specific tool to be used in the current scope.
    """
    rpstack = getrpstack(recdict, line_nr)
    args = str2dictlist(rpstack,
                  expand(line_nr, recdict, arg, Expand(0, Expand.quote_aap)))
    if len(args) != 1:
        recipe_error(rpstack, _(':usetool requires one argument'))

    toolname = args[0]["name"]
    try:
        # Only import "tools_name" module from specific directories.
        tools_path = [ os.path.join(d, "tools") 
                       for d in get_import_dirs(recdict) ]
        fpd = imp.find_module(toolname, tools_path)
        exec "tools_%s = imp.load_module(toolname, *fpd)" % toolname
    except:
        recipe_error(rpstack, _('Tool "%s" is not supported') % toolname)

    if eval("tools_%s.exists()" % toolname):
        exec "tools_%s.define_actions()" % toolname
        exec "tools_%s.use_actions(recdict)" % toolname
    else:
        recipe_error(rpstack, _('Tool "%s" cannot be found on the system')
                                                                    % toolname)


def aap_progsearch(line_nr, recdict, arg):
    """Search for programs in $PATH."""
    # Get the arguments, first one is the variable name.
    rpstack = getrpstack(recdict, line_nr)
    args = str2dictlist(rpstack,
                  expand(line_nr, recdict, arg, Expand(0, Expand.quote_aap)))
    if len(args) < 2:
        recipe_error(rpstack, _(':progsearch requires at least two arguments'))
    assert_var_name(rpstack, args[0]["name"])

    # Separate scope and decide which dictionary to use.
    rd, scope, varname = get_scope_recdict(rpstack, recdict, args[0]["name"])

    from RecPython import program_path

    # Search for the programs, quit as soon as one is found.
    prog = ''
    for arg in args[1:]:
        prog = program_path(arg["name"])
        if prog:
            break

    if not prog:
        msg_note(recdict, _(':progsearch did not find any of %s')
                                          % map(lambda x: x["name"], args[1:]))

    # If the program name includes a space put it in double quotes.
    elif " " in prog:
        prog = '"%s"' % prog

    try:
        rd[varname] = prog
    except StandardError, e:
        recipe_error(rpstack, _(':progsearch assignment error: %s') % str(e))


def aap_do(line_nr, recdict, arg):
    """Execute an action for a type of file: :do action {attr} fname ..."""
    rpstack = getrpstack(recdict, line_nr)

    args = str2dictlist(rpstack,
                  expand(line_nr, recdict, arg, Expand(1, Expand.quote_aap)))
    if len(args) < 2:
        recipe_error(rpstack, _(':do requires at least two arguments'))
    args = [args[0]] + dictlist_expand(args[1:])

    from Action import action_run
    
    try:
        msg = action_run(recdict, args)
    finally:
        # When "remove" used delete all the arguments.
        if args[0].get("remove"):
            for arg in args[1:]:
                try:
                    os.remove(arg["name"])
                except StandardError, e:
                    msg_warning(recdict, _('Could not remove "%s": %s')
                                                      % (arg["name"], str(e)))

    if msg:
        recipe_error(rpstack, msg)


def aap_exit(line_nr, recdict, arg):
    """Quit aap."""
    aap_quit(line_nr, recdict, arg)

def aap_quit(line_nr, recdict, arg):
    """Quit aap."""
    if len(arg) > 0:
        raise NormalExit, int(arg)
    raise NormalExit

def aap_finish(line_nr, recdict, arg):
    """Quit the current recipe."""
    # Throw an exception to cancel executing the rest of the script
    # generated from the recipe.  This is catched in read_recipe()
    raise FinishRecipe


def aap_proxy(line_nr, recdict, arg):
    """Specify a proxy server."""
    rpstack = getrpstack(recdict, line_nr)

    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg)
    if optiondict:
        recipe_error(rpstack, _(":proxy command does not accept options"))
    if not argdictlist:
        recipe_error(rpstack, _(":proxy command requires a file argument"))

    if len(argdictlist) == 1:
        n = "HTTP"
    elif len(argdictlist) == 2:
        n = string.upper(argdictlist[0]["name"])
        if n != "HTTP" and n != "FTP" and n != "GOPHER":
            recipe_error(rpstack, _(':proxy argument must be "http", "ftp" or "gopher"; "%s" is not accepted') % argdictlist[0]["name"])
    else:
        recipe_error(rpstack, _("Too many arguments for :proxy command"))

    n = n + "_PROXY"
    os.environ[n] = argdictlist[1]["name"]


def aap_checksum(line_nr, recdict, arg):
    """Compute checksum and compare with value."""
    rpstack = getrpstack(recdict, line_nr)

    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg)
    if not argdictlist:
        recipe_error(rpstack, _(":checksum command requires a file argument"))

    from Sign import check_md5

    # Loop over all items, adding attributes to the node.
    for i in argdictlist:
        name = i["name"]
        if not os.path.exists(name):
            msg_note(recdict, _(':checksum: file does not exists: "%s"')
                                                                        % name)
        else:
            if not i.get("md5"):
                recipe_error(rpstack, _('md5 attribute missing for "%s"')
                                                                        % name)
            md5 = check_md5(recdict, name)
            if md5 == "unknown":
                recipe_error(rpstack, _('cannot compute md5 checksum for "%s"')
                                                                        % name)
            if md5 != i.get("md5"):
                recipe_error(rpstack, _('md5 checksum mismatch for  "%s"')
                                                                        % name)

def aap_mkdownload(line_nr, recdict, arg):
    """Generate a recipe for downloading files."""
    rpstack = getrpstack(recdict, line_nr)
    work = getwork(recdict)

    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg)
    if len(argdictlist) < 2:
        recipe_error(rpstack,
                  _(":mkdownload command requires a recipe and file argument"))

    rname = os.path.expanduser(argdictlist[0]["name"])
    try:
        fp = open(rname, "w")
    except IOError, e:
        msg_error(recdict, (_('Cannot open file for writing "%s": ')
                                                             % rname) + str(e))
    write_error = _('Cannot write to file "%s": ')
    try:
        fp.write('# This recipe was generated with ":mkdownload".\n')
        fetch = argdictlist[0].get("fetch")
        if fetch:
            fp.write(":recipe {fetch = %s}\n" % fetch)
        fp.write("all fetch:\n")
    except IOError, e:
        msg_error(recdict, (write_error % rname) + str(e))

    from RecPython import get_md5

    # Expand wildcards.
    xargs = dictlist_expand(argdictlist[1:])

    # loop over all file arguments
    dirs = []
    prev_fetch = ''
    for afile in xargs:
        fname = afile["name"]
        if not os.path.exists(fname):
            recipe_error(rpstack,
                        _(':mkdownload argument does not exist: "%s"') % fname)
        fetch = afile.get("fetch")
        if not fetch:
            node = work.find_node(fname)
            if node:
                fetch = node.attributes.get("fetch")
            if not fetch:
                recipe_error(rpstack,
                        _(':mkdownload argument without fetch attribute: "%s"')
                                                                       % fname)
        try:
            adir = os.path.dirname(fname)
            if adir and not adir in dirs:
                fp.write("  :mkdir {f} %s\n" % listitem2str(adir))
                dirs.append(adir)
            fp.write("  file = %s\n" % listitem2str(fname))
            if fetch != prev_fetch:
                fp.write("  fetcha = %s\n" % listitem2str(fetch))
                prev_fetch = fetch
            fp.write('  @if get_md5(file) != "%s":\n' % get_md5(fname))
            fp.write('    :fetch {fetch = $fetcha} $file\n')
        except IOError, e:
            msg_error(recdict, (write_error % rname) + str(e))

    try:
        fp.close()
    except IOError, e:
        msg_error(recdict, (write_error % rname) + str(e))


def aap_cd(line_nr, recdict, arg):
    """:cd command"""
    aap_chdir(line_nr, recdict, arg, cmd = "cd")

def aap_chdir(line_nr, recdict, arg, cmd = "chdir"):
    """:chdir command"""
    rpstack = getrpstack(recdict, line_nr)
    varlist = str2dictlist(rpstack,
                    expand(line_nr, recdict, arg, Expand(1, Expand.quote_aap)))
    if len(varlist) < 1:
        recipe_error(rpstack, _(":%s command requires at least one argument")
                                                                         % cmd)
    dictlist_expanduser(varlist)

    # Concatenate all arguments inserting "/" where needed.
    adir = varlist[0]["name"]
    i = 1
    while i < len(varlist):
        adir = os.path.join(adir, varlist[i]["name"])
        i = i + 1

    if adir == '-':
        if not recdict["_prevdir"]:
            recipe_error(rpstack, _("No previous directory for :%s -") % cmd)
        adir = recdict["_prevdir"]
    if cmd == "cd" or cmd == "chdir":
        recdict["_prevdir"] = os.getcwd()
    try:
        os.chdir(adir)
    except StandardError, e:
        recipe_error(rpstack, (_(':%s "%s": ') % (cmd, adir)) + str(e))
    msg_changedir(recdict, os.path.abspath(os.getcwd()))

def aap_pushdir(line_nr, recdict, arg):
    """:pushdir command"""
    recdict["_dirstack"].append(os.getcwd())
    aap_chdir(line_nr, recdict, arg, cmd = "pushdir")

def aap_popdir(line_nr, recdict, arg):
    """:popdir command"""
    rpstack = getrpstack(recdict, line_nr)
    if arg:
        recipe_error(rpstack, _(':popdir does not take an argument'))
    if not recdict["_dirstack"]:
        recipe_error(rpstack, _(':popdir: directory stack is empty'))

    adir = recdict["_dirstack"].pop()
    try:
        os.chdir(adir)
    except StandardError, e:
        recipe_error(rpstack, (_(':popdir to "%s": ') % adir) + str(e))
    msg_changedir(recdict, os.path.abspath(adir))


def aap_assertpkg(line_nr, recdict, arg):
    """
    :assertpkg command
    """
    aap_installpkg(line_nr, recdict, arg, cmdname = "assertpkg")

def aap_installpkg(line_nr, recdict, arg, cmdname = "installpkg"):
    """
    :installpkg command
    Also used for :assertpkg command.
    """
    rpstack = getrpstack(recdict, line_nr)
    varlist = str2dictlist(rpstack,
                    expand(line_nr, recdict, arg, Expand(1, Expand.quote_aap)))
    if len(varlist) == 0:
        recipe_error(rpstack, _(":%s command requires an argument") % cmdname)

    from DoInstall import assert_pkg, install_pkg

    for pkg in varlist:
        if pkg["name"] == "":
            recipe_error(rpstack, _("Empty item in :%s") % cmdname)
        if cmdname == "assertpkg":
            assert_pkg(rpstack, recdict, pkg["name"],
                                                optional = pkg.get("optional"))
        else:
            install_pkg(rpstack, recdict, pkg["name"])


def aap_asroot(line_nr, recdict, arg):
    """:asroot command"""
    # Behave like ":system" on non-Unix systems (there is no super-user)
    # or when we can write in "/" (already super-user).
    if os.name != "posix" or os.access("/", os.W_OK):
        aap_system(line_nr, recdict, arg)
        return

    rpstack = getrpstack(recdict, line_nr)
    cmd = expand(line_nr, recdict, arg, Expand(0, Expand.quote_shell))
    if not cmd:
        recipe_error(rpstack, _(":asroot command requires an argument"))

    recdict['sysresult'] = 1
    r = do_as_root(recdict, rpstack, cmd)
    if r == 0:
        recipe_error(rpstack, _(":asroot command failed"))
    if r == 1:
        # Success!
        recdict['sysresult'] = 0


didwarn = 0

def aap_conf(line_nr, recdict, arg):
    """
    :conf testname [args...]
    """
    global didwarn
    if not didwarn:
        msg_warning(recdict, "The :conf command is experimental, it may change in future versions")
        didwarn = 1

    rpstack = getrpstack(recdict, line_nr)
    optiondict, attrdict, argdictlist = get_args(line_nr, recdict, arg,
                                    {"required": "required", "oneof": "oneof"})
    if len(argdictlist) < 1:
        recipe_error(rpstack,
                  _(":conf command requires a test name argument"))
    if attrdict:
        option_error(rpstack, attrdict, ":conf")

    # The implementation is quite a bit of code, it's in a separate module.
    from DoConf import doconf
    doconf(line_nr, recdict, optiondict, argdictlist)


# vim: set sw=4 et sts=4 tw=79 fo+=l: