From 553ca1e649293ef87e96dd3e7621fd87e0b59986 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Tue, 29 Aug 2023 12:41:46 +0200 Subject: [PATCH] Update to waf 2.0.26 This makes waf compatible with Python 3.12 again. Also, apply modifications needed for MacOS and add as a patch file (see commits 0f2e3b2 and dc6c995). Signed-off-by: Nils Philippsen Upstream: https:\/\/github.com\/jackaudio\/jack2\/commit\/250420381b1a6974798939ad7104ab1a4b9a9994 Signed-off-by: Fabrice Fontaine --- waf | 7 +- waflib-macos-mods.patch | 18 + waflib\/Build.py | 47 +- waflib\/ConfigSet.py | 2 +- waflib\/Configure.py | 46 +- waflib\/Context.py | 24 +- waflib\/Logs.py | 9 +- waflib\/Node.py | 3 +- waflib\/Options.py | 31 +- waflib\/Runner.py | 27 +- waflib\/Scripting.py | 27 +- waflib\/Task.py | 48 ++- waflib\/TaskGen.py | 16 +- waflib\/Tools\/c_aliases.py | 10 +- waflib\/Tools\/c_config.py | 37 +- waflib\/Tools\/c_preproc.py | 6 +- waflib\/Tools\/c_tests.py | 18 +- waflib\/Tools\/ccroot.py | 20 +- waflib\/Tools\/compiler_c.py | 25 +- waflib\/Tools\/compiler_cxx.py | 25 +- waflib\/Tools\/irixcc.py | 14 +- waflib\/Tools\/msvc.py | 45 +- waflib\/Tools\/waf_unit_test.py | 14 +- waflib\/Utils.py | 60 ++- waflib\/ansiterm.py | 2 +- waflib\/extras\/clang_cross.py | 92 ++++ waflib\/extras\/clang_cross_common.py | 113 +++++ waflib\/extras\/clangxx_cross.py | 106 +++++ waflib\/extras\/classic_runner.py | 68 +++ waflib\/extras\/color_msvc.py | 59 +++ waflib\/extras\/fc_fujitsu.py | 52 +++ waflib\/extras\/fc_nfort.py | 52 +++ waflib\/extras\/genpybind.py | 194 +++++++++ waflib\/extras\/haxe.py | 154 +++++++ waflib\/extras\/msvc_pdb.py | 46 ++ waflib\/extras\/sphinx.py | 120 ++++++ waflib\/extras\/wafcache.py | 648 ++++++++++++++++++++++++++++ waflib\/extras\/xcode6.py | 18 +- waflib\/fixpy2.py | 2 +- waflib\/processor.py | 4 + 40 files changed, 2114 insertions(+), 195 deletions(-) create mode 100644 waflib-macos-mods.patch create mode 100644 waflib\/extras\/clang_cross.py create mode 100644 waflib\/extras\/clang_cross_common.py create mode 100644 waflib\/extras\/clangxx_cross.py create mode 100644 waflib\/extras\/classic_runner.py create mode 100644 waflib\/extras\/color_msvc.py create mode 100644 waflib\/extras\/fc_fujitsu.py create mode 100644 waflib\/extras\/fc_nfort.py create mode 100644 waflib\/extras\/genpybind.py create mode 100644 waflib\/extras\/haxe.py create mode 100644 waflib\/extras\/msvc_pdb.py create mode 100644 waflib\/extras\/sphinx.py create mode 100644 waflib\/extras\/wafcache.py diff --git a\/waf b\/waf index 845fba5e9..38b2c9106 100755 --- a\/waf +++ b\/waf @@ -1,4 +1,4 @@ -#!\/usr\/bin\/python3 +#!\/usr\/bin\/env python # encoding: latin-1 # Thomas Nagy, 2005-2018 # @@ -32,7 +32,7 @@ POSSIBILITY OF SUCH DAMAGE. import os, sys, inspect -VERSION="2.0.12" +VERSION="2.0.26" REVISION="x" GIT="x" INSTALL="x" @@ -142,6 +142,9 @@ def find_lib(): if name.endswith('waf-light'): w = test(base) if w: return w + for dir in sys.path: + if test(dir): + return dir err('waf-light requires waflib -> export WAFDIR=\/folder') dirname = '%s-%s-%s' % (WAF, VERSION, REVISION) diff --git a\/waflib-macos-mods.patch b\/waflib-macos-mods.patch new file mode 100644 index 000000000..9e2c8a3de --- \/dev\/null +++ b\/waflib-macos-mods.patch @@ -0,0 +1,18 @@ +diff --git a\/waflib\/Tools\/ccroot.py b\/waflib\/Tools\/ccroot.py +index cfef8bf5..484846f5 100644 +--- a\/waflib\/Tools\/ccroot.py ++++ b\/waflib\/Tools\/ccroot.py +@@ -575,12 +575,10 @@ def apply_vnum(self): + + cnum = getattr(self, 'cnum', str(nums[0])) + cnums = cnum.split('.') +- if len(cnums)>len(nums) or nums[0:len(cnums)] != cnums: +- raise Errors.WafError('invalid compatibility version %s' % cnum) + + libname = node.name + if libname.endswith('.dylib'): +- name3 = libname.replace('.dylib', '.%s.dylib' % self.vnum) ++ name3 = libname.replace('.dylib', '.%s.dylib' % cnums[0]) + name2 = libname.replace('.dylib', '.%s.dylib' % cnum) + else: + name3 = libname + '.' + self.vnum diff --git a\/waflib\/Build.py b\/waflib\/Build.py index c9661df15..b49dd8302 100644 --- a\/waflib\/Build.py +++ b\/waflib\/Build.py @@ -104,7 +104,7 @@ def __init__(self, **kw): """Amount of jobs to run in parallel""" self.targets = Options.options.targets - """List of targets to build (default: \*)""" + """List of targets to build (default: \\*)""" self.keep = Options.options.keep """Whether the build should continue past errors""" @@ -753,10 +753,12 @@ def tgpost(tg): else: ln = self.launch_node() if ln.is_child_of(self.bldnode): - Logs.warn('Building from the build directory, forcing --targets=*') + if Logs.verbose > 1: + Logs.warn('Building from the build directory, forcing --targets=*') ln = self.srcnode elif not ln.is_child_of(self.srcnode): - Logs.warn('CWD %s is not under %s, forcing --targets=* (run distclean?)', ln.abspath(), self.srcnode.abspath()) + if Logs.verbose > 1: + Logs.warn('CWD %s is not under %s, forcing --targets=* (run distclean?)', ln.abspath(), self.srcnode.abspath()) ln = self.srcnode def is_post(tg, ln): @@ -1054,7 +1056,7 @@ def post_run(self): def get_install_path(self, destdir=True): """ Returns the destination path where files will be installed, pre-pending `destdir`. - + Relative paths will be interpreted relative to `PREFIX` if no `destdir` is given. :rtype: string @@ -1062,11 +1064,11 @@ def get_install_path(self, destdir=True): if isinstance(self.install_to, Node.Node): dest = self.install_to.abspath() else: - dest = Utils.subst_vars(self.install_to, self.env) + dest = os.path.normpath(Utils.subst_vars(self.install_to, self.env)) if not os.path.isabs(dest): - dest = os.path.join(self.env.PREFIX, dest) + dest = os.path.join(self.env.PREFIX, dest) if destdir and Options.options.destdir: - dest = os.path.join(Options.options.destdir, os.path.splitdrive(dest)[1].lstrip(os.sep)) + dest = Options.options.destdir.rstrip(os.sep) + os.sep + os.path.splitdrive(dest)[1].lstrip(os.sep) return dest def copy_fun(self, src, tgt): @@ -1160,11 +1162,19 @@ def do_install(self, src, tgt, lbl, **kw): # same size and identical timestamps -> make no copy if st1.st_mtime + 2 >= st2.st_mtime and st1.st_size == st2.st_size: if not self.generator.bld.progress_bar: - Logs.info('- install %s (from %s)', tgt, lbl) + + c1 = Logs.colors.NORMAL + c2 = Logs.colors.BLUE + + Logs.info('%s- install %s%s%s (from %s)', c1, c2, tgt, c1, lbl) return False if not self.generator.bld.progress_bar: - Logs.info('+ install %s (from %s)', tgt, lbl) + + c1 = Logs.colors.NORMAL + c2 = Logs.colors.BLUE + + Logs.info('%s+ install %s%s%s (from %s)', c1, c2, tgt, c1, lbl) # Give best attempt at making destination overwritable, # like the 'install' utility used by 'make install' does. @@ -1221,14 +1231,18 @@ def do_link(self, src, tgt, **kw): """ if os.path.islink(tgt) and os.readlink(tgt) == src: if not self.generator.bld.progress_bar: - Logs.info('- symlink %s (to %s)', tgt, src) + c1 = Logs.colors.NORMAL + c2 = Logs.colors.BLUE + Logs.info('%s- symlink %s%s%s (to %s)', c1, c2, tgt, c1, src) else: try: os.remove(tgt) except OSError: pass if not self.generator.bld.progress_bar: - Logs.info('+ symlink %s (to %s)', tgt, src) + c1 = Logs.colors.NORMAL + c2 = Logs.colors.BLUE + Logs.info('%s+ symlink %s%s%s (to %s)', c1, c2, tgt, c1, src) os.symlink(src, tgt) self.fix_perms(tgt) @@ -1237,7 +1251,9 @@ def do_uninstall(self, src, tgt, lbl, **kw): See :py:meth:`waflib.Build.inst.do_install` """ if not self.generator.bld.progress_bar: - Logs.info('- remove %s', tgt) + c1 = Logs.colors.NORMAL + c2 = Logs.colors.BLUE + Logs.info('%s- remove %s%s%s', c1, c2, tgt, c1) #self.uninstall.append(tgt) try: @@ -1257,7 +1273,9 @@ def do_unlink(self, src, tgt, **kw): """ try: if not self.generator.bld.progress_bar: - Logs.info('- remove %s', tgt) + c1 = Logs.colors.NORMAL + c2 = Logs.colors.BLUE + Logs.info('%s- remove %s%s%s', c1, c2, tgt, c1) os.remove(tgt) except OSError: pass @@ -1318,7 +1336,8 @@ def build(bld): lst = [] for env in self.all_envs.values(): lst.extend(self.root.find_or_declare(f) for f in env[CFG_FILES]) - for n in self.bldnode.ant_glob('**\/*', excl='.lock* *conf_check_*\/** config.log c4che\/*', quiet=True): + excluded_dirs = '.lock* *conf_check_*\/** config.log %s\/*' % CACHE_DIR + for n in self.bldnode.ant_glob('**\/*', excl=excluded_dirs, quiet=True): if n in lst: continue n.delete() diff --git a\/waflib\/ConfigSet.py b\/waflib\/ConfigSet.py index 84736c9c8..901fba6c0 100644 --- a\/waflib\/ConfigSet.py +++ b\/waflib\/ConfigSet.py @@ -11,7 +11,7 @@ import copy, re, os from waflib import Logs, Utils -re_imp = re.compile('^(#)*?([^#=]*?)\ =\ (.*?)$', re.M) +re_imp = re.compile(r'^(#)*?([^#=]*?)\ =\ (.*?)$', re.M) class ConfigSet(object): """ diff --git a\/waflib\/Configure.py b\/waflib\/Configure.py index d0a4793a8..f6fdc4e94 100644 --- a\/waflib\/Configure.py +++ b\/waflib\/Configure.py @@ -125,7 +125,7 @@ def init_dirs(self): self.bldnode.mkdir() if not os.path.isdir(self.bldnode.abspath()): - conf.fatal('Could not create the build directory %s' % self.bldnode.abspath()) + self.fatal('Could not create the build directory %s' % self.bldnode.abspath()) def execute(self): """ @@ -180,6 +180,7 @@ def execute(self): env.hash = self.hash env.files = self.files env.environ = dict(self.environ) + env.launch_dir = Context.launch_dir if not (self.env.NO_LOCK_IN_RUN or env.environ.get('NO_LOCK_IN_RUN') or getattr(Options.options, 'no_lock_in_run')): env.store(os.path.join(Context.run_dir, Options.lockfile)) @@ -438,7 +439,7 @@ def find_program(self, filename, **kw): var = kw.get('var', '') if not var: - var = re.sub(r'[-.]', '_', filename[0].upper()) + var = re.sub(r'\W', '_', filename[0].upper()) path_list = kw.get('path_list', '') if path_list: @@ -507,23 +508,27 @@ def find_binary(self, filenames, exts, paths): @conf def run_build(self, *k, **kw): """ - Create a temporary build context to execute a build. A reference to that build - context is kept on self.test_bld for debugging purposes, and you should not rely - on it too much (read the note on the cache below). - The parameters given in the arguments to this function are passed as arguments for - a single task generator created in the build. Only three parameters are obligatory: + Create a temporary build context to execute a build. A temporary reference to that build + context is kept on self.test_bld for debugging purposes. + The arguments to this function are passed to a single task generator for that build. + Only three parameters are mandatory: :param features: features to pass to a task generator created in the build :type features: list of string :param compile_filename: file to create for the compilation (default: *test.c*) :type compile_filename: string - :param code: code to write in the filename to compile + :param code: input file contents :type code: string - Though this function returns *0* by default, the build may set an attribute named *retval* on the + Though this function returns *0* by default, the build may bind attribute named *retval* on the build context object to return a particular value. See :py:func:`waflib.Tools.c_config.test_exec_fun` for example. - This function also provides a limited cache. To use it, provide the following option:: + The temporary builds creates a temporary folder; the name of that folder is calculated + by hashing input arguments to this function, with the exception of :py:class:`waflib.ConfigSet.ConfigSet` + objects which are used for both reading and writing values. + + This function also features a cache which is disabled by default; that cache relies + on the hash value calculated as indicated above:: def options(opt): opt.add_option('--confcache', dest='confcache', default=0, @@ -534,10 +539,24 @@ def options(opt): $ waf configure --confcache """ - lst = [str(v) for (p, v) in kw.items() if p != 'env'] - h = Utils.h_list(lst) + buf = [] + for key in sorted(kw.keys()): + v = kw[key] + if isinstance(v, ConfigSet.ConfigSet): + # values are being written to, so they are excluded from contributing to the hash + continue + elif hasattr(v, '__call__'): + buf.append(Utils.h_fun(v)) + else: + buf.append(str(v)) + h = Utils.h_list(buf) dir = self.bldnode.abspath() + os.sep + (not Utils.is_win32 and '.' or '') + 'conf_check_' + Utils.to_hex(h) + cachemode = kw.get('confcache', getattr(Options.options, 'confcache', None)) + + if not cachemode and os.path.exists(dir): + shutil.rmtree(dir) + try: os.makedirs(dir) except OSError: @@ -548,7 +567,6 @@ def options(opt): except OSError: self.fatal('cannot use the configuration test folder %r' % dir) - cachemode = getattr(Options.options, 'confcache', None) if cachemode == 1: try: proj = ConfigSet.ConfigSet(os.path.join(dir, 'cache_run_build')) @@ -588,7 +606,7 @@ def options(opt): else: ret = getattr(bld, 'retval', 0) finally: - if cachemode == 1: + if cachemode: # cache the results each time proj = ConfigSet.ConfigSet() proj['cache_run_build'] = ret diff --git a\/waflib\/Context.py b\/waflib\/Context.py index 761b521f5..369664819 100644 --- a\/waflib\/Context.py +++ b\/waflib\/Context.py @@ -6,20 +6,30 @@ Classes and functions enabling the command system """ -import os, re, imp, sys +import os, re, sys from waflib import Utils, Errors, Logs import waflib.Node +if sys.hexversion > 0x3040000: + import types + class imp(object): + new_module = lambda x: types.ModuleType(x) +else: + import imp + # the following 3 constants are updated on each new release (do not touch) -HEXVERSION=0x2000c00 +HEXVERSION=0x2001a00 """Constant updated on new releases""" -WAFVERSION="2.0.12" +WAFVERSION="2.0.26" """Constant updated on new releases""" -WAFREVISION="54841218840ffa34fddf834680a5a17db69caa12" +WAFREVISION="0fb985ce1932c6f3e7533f435e4ee209d673776e" """Git revision when the waf version is updated""" +WAFNAME="waf" +"""Application name displayed on --help""" + ABI = 20 """Version of the build data cache file format (used in :py:const:`waflib.Context.DBFILE`)""" @@ -134,7 +144,7 @@ def foo(ctx): :type fun: string .. inheritance-diagram:: waflib.Context.Context waflib.Build.BuildContext waflib.Build.InstallContext waflib.Build.UninstallContext waflib.Build.StepContext waflib.Build.ListContext waflib.Configure.ConfigurationContext waflib.Scripting.Dist waflib.Scripting.DistCheck waflib.Build.CleanContext - + :top-classes: waflib.Context.Context """ errors = Errors @@ -613,7 +623,7 @@ def load_special_tools(self, var, ban=[]): is typically called once for a programming language group, see for example :py:mod:`waflib.Tools.compiler_c` - :param var: glob expression, for example 'cxx\_\*.py' + :param var: glob expression, for example 'cxx\\_\\*.py' :type var: string :param ban: list of exact file names to exclude :type ban: list of string @@ -678,7 +688,7 @@ def load_module(path, encoding=None): def load_tool(tool, tooldir=None, ctx=None, with_sys_path=True): """ - Importx a Waf tool as a python module, and stores it in the dict :py:const:`waflib.Context.Context.tools` + Imports a Waf tool as a python module, and stores it in the dict :py:const:`waflib.Context.Context.tools` :type tool: string :param tool: Name of the tool diff --git a\/waflib\/Logs.py b\/waflib\/Logs.py index 2a475169b..298411db5 100644 --- a\/waflib\/Logs.py +++ b\/waflib\/Logs.py @@ -237,7 +237,10 @@ def format(self, rec): if rec.levelno >= logging.INFO: # the goal of this is to format without the leading "Logs, hour" prefix if rec.args: - return msg % rec.args + try: + return msg % rec.args + except UnicodeDecodeError: + return msg.encode('utf-8') % rec.args return msg rec.msg = msg @@ -276,9 +279,9 @@ def error(*k, **kw): def warn(*k, **kw): """ - Wraps logging.warn + Wraps logging.warning """ - log.warn(*k, **kw) + log.warning(*k, **kw) def info(*k, **kw): """ diff --git a\/waflib\/Node.py b\/waflib\/Node.py index 4ac1ea8a0..2ad184669 100644 --- a\/waflib\/Node.py +++ b\/waflib\/Node.py @@ -73,7 +73,7 @@ def ant_matcher(s, ignorecase): if k == '**': accu.append(k) else: - k = k.replace('.', '[.]').replace('*','.*').replace('?', '.').replace('+', '\\+') + k = k.replace('.', '[.]').replace('*', '.*').replace('?', '.').replace('+', '\\+') k = '^%s$' % k try: exp = re.compile(k, flags=reflags) @@ -595,7 +595,6 @@ def ant_iter(self, accept=None, maxdepth=25, pats=[], dir=False, src=True, remov :rtype: iterator """ dircont = self.listdir() - dircont.sort() try: lst = set(self.children.keys()) diff --git a\/waflib\/Options.py b\/waflib\/Options.py index ad802d4b9..d4104917c 100644 --- a\/waflib\/Options.py +++ b\/waflib\/Options.py @@ -44,7 +44,7 @@ class opt_parser(optparse.OptionParser): """ def __init__(self, ctx, allow_unknown=False): optparse.OptionParser.__init__(self, conflict_handler='resolve', add_help_option=False, - version='waf %s (%s)' % (Context.WAFVERSION, Context.WAFREVISION)) + version='%s %s (%s)' % (Context.WAFNAME, Context.WAFVERSION, Context.WAFREVISION)) self.formatter.width = Logs.get_term_cols() self.ctx = ctx self.allow_unknown = allow_unknown @@ -62,6 +62,21 @@ def _process_args(self, largs, rargs, values): else: self.error(str(e)) + def _process_long_opt(self, rargs, values): + # --custom-option=-ftxyz is interpreted as -f -t... see #2280 + if self.allow_unknown: + back = [] + rargs + try: + optparse.OptionParser._process_long_opt(self, rargs, values) + except optparse.BadOptionError: + while rargs: + rargs.pop() + rargs.extend(back) + rargs.pop(0) + raise + else: + optparse.OptionParser._process_long_opt(self, rargs, values) + def print_usage(self, file=None): return self.print_help(file) @@ -96,11 +111,11 @@ def get_usage(self): lst.sort() ret = '\n'.join(lst) - return '''waf [commands] [options] + return '''%s [commands] [options] -Main commands (example: .\/waf build -j4) +Main commands (example: .\/%s build -j4) %s -''' % ret +''' % (Context.WAFNAME, Context.WAFNAME, ret) class OptionsContext(Context.Context): @@ -141,9 +156,9 @@ def __init__(self, **kw): gr.add_option('-o', '--out', action='store', default='', help='build dir for the project', dest='out') gr.add_option('-t', '--top', action='store', default='', help='src dir for the project', dest='top') - gr.add_option('--no-lock-in-run', action='store_true', default='', help=optparse.SUPPRESS_HELP, dest='no_lock_in_run') - gr.add_option('--no-lock-in-out', action='store_true', default='', help=optparse.SUPPRESS_HELP, dest='no_lock_in_out') - gr.add_option('--no-lock-in-top', action='store_true', default='', help=optparse.SUPPRESS_HELP, dest='no_lock_in_top') + gr.add_option('--no-lock-in-run', action='store_true', default=os.environ.get('NO_LOCK_IN_RUN', ''), help=optparse.SUPPRESS_HELP, dest='no_lock_in_run') + gr.add_option('--no-lock-in-out', action='store_true', default=os.environ.get('NO_LOCK_IN_OUT', ''), help=optparse.SUPPRESS_HELP, dest='no_lock_in_out') + gr.add_option('--no-lock-in-top', action='store_true', default=os.environ.get('NO_LOCK_IN_TOP', ''), help=optparse.SUPPRESS_HELP, dest='no_lock_in_top') default_prefix = getattr(Context.g_module, 'default_prefix', os.environ.get('PREFIX')) if not default_prefix: @@ -282,6 +297,8 @@ def parse_cmd_args(self, _args=None, cwd=None, allow_unknown=False): elif arg != 'options': commands.append(arg) + if options.jobs < 1: + options.jobs = 1 for name in 'top out destdir prefix bindir libdir'.split(): # those paths are usually expanded from Context.launch_dir if getattr(options, name, None): diff --git a\/waflib\/Runner.py b\/waflib\/Runner.py index 261084d27..350c86a22 100644 --- a\/waflib\/Runner.py +++ b\/waflib\/Runner.py @@ -37,6 +37,8 @@ def __len__(self): return len(self.lst) def __iter__(self): return iter(self.lst) + def __str__(self): + return 'PriorityTasks: [%s]' % '\n '.join(str(x) for x in self.lst) def clear(self): self.lst = [] def append(self, task): @@ -69,7 +71,7 @@ def __init__(self, spawner, task): """Task to execute""" self.spawner = spawner """Coordinator object""" - self.setDaemon(1) + self.daemon = True self.start() def run(self): """ @@ -96,7 +98,7 @@ def __init__(self, master): """:py:class:`waflib.Runner.Parallel` producer instance""" self.sem = Utils.threading.Semaphore(master.numjobs) """Bounded semaphore that prevents spawning more than *n* concurrent consumers""" - self.setDaemon(1) + self.daemon = True self.start() def run(self): """ @@ -181,10 +183,12 @@ def __init__(self, bld, j=2): The reverse dependency graph of dependencies obtained from Task.run_after """ - self.spawner = Spawner(self) + self.spawner = None """ Coordinating daemon thread that spawns thread consumers """ + if self.numjobs > 1: + self.spawner = Spawner(self) def get_next_task(self): """ @@ -254,6 +258,8 @@ def refill_task_list(self): self.outstanding.append(x) break else: + if self.stop or self.error: + break raise Errors.WafError('Broken revdeps detected on %r' % self.incomplete) else: tasks = next(self.biter) @@ -331,11 +337,16 @@ def try_unfreeze(x): if hasattr(tsk, 'semaphore'): sem = tsk.semaphore - sem.release(tsk) - while sem.waiting and not sem.is_locked(): - # take a frozen task, make it ready to run - x = sem.waiting.pop() - self._add_task(x) + try: + sem.release(tsk) + except KeyError: + # TODO + pass + else: + while sem.waiting and not sem.is_locked(): + # take a frozen task, make it ready to run + x = sem.waiting.pop() + self._add_task(x) def get_out(self): """ diff --git a\/waflib\/Scripting.py b\/waflib\/Scripting.py index 749d4f2e6..a80cb3678 100644 --- a\/waflib\/Scripting.py +++ b\/waflib\/Scripting.py @@ -216,7 +216,10 @@ def parse_options(): ctx = Context.create_context('options') ctx.execute() if not Options.commands: - Options.commands.append(default_cmd) + if isinstance(default_cmd, list): + Options.commands.extend(default_cmd) + else: + Options.commands.append(default_cmd) if Options.options.whelp: ctx.parser.print_help() sys.exit(0) @@ -280,7 +283,7 @@ def distclean_dir(dirname): pass try: - shutil.rmtree('c4che') + shutil.rmtree(Build.CACHE_DIR) except OSError: pass @@ -303,7 +306,7 @@ def remove_and_log(k, fun): # remove a build folder, if any cur = '.' - if ctx.options.no_lock_in_top: + if os.environ.get('NO_LOCK_IN_TOP') or ctx.options.no_lock_in_top: cur = ctx.options.out try: @@ -329,7 +332,12 @@ def remove_and_log(k, fun): else: remove_and_log(env.out_dir, shutil.rmtree) - for k in (env.out_dir, env.top_dir, env.run_dir): + env_dirs = [env.out_dir] + if not (os.environ.get('NO_LOCK_IN_TOP') or ctx.options.no_lock_in_top): + env_dirs.append(env.top_dir) + if not (os.environ.get('NO_LOCK_IN_RUN') or ctx.options.no_lock_in_run): + env_dirs.append(env.run_dir) + for k in env_dirs: p = os.path.join(k, Options.lockfile) remove_and_log(p, os.remove) @@ -380,7 +388,11 @@ def archive(self): for x in files: archive_name = self.get_base_name() + '\/' + x.path_from(self.base_path) - zip.write(x.abspath(), archive_name, zipfile.ZIP_DEFLATED) + if os.environ.get('SOURCE_DATE_EPOCH'): + # TODO: parse that timestamp + zip.writestr(zipfile.ZipInfo(archive_name), x.read(), zipfile.ZIP_DEFLATED) + else: + zip.write(x.abspath(), archive_name, zipfile.ZIP_DEFLATED) zip.close() else: self.fatal('Valid algo types are tar.bz2, tar.gz, tar.xz or zip') @@ -417,6 +429,8 @@ def add_tar_file(self, x, tar): tinfo.gid = 0 tinfo.uname = 'root' tinfo.gname = 'root' + if os.environ.get('SOURCE_DATE_EPOCH'): + tinfo.mtime = int(os.environ.get('SOURCE_DATE_EPOCH')) if os.path.isfile(p): with open(p, 'rb') as f: @@ -598,12 +612,15 @@ def execute(self): cmd = env.config_cmd or 'configure' if Configure.autoconfig == 'clobber': tmp = Options.options.__dict__ + launch_dir_tmp = Context.launch_dir if env.options: Options.options.__dict__ = env.options + Context.launch_dir = env.launch_dir try: run_command(cmd) finally: Options.options.__dict__ = tmp + Context.launch_dir = launch_dir_tmp else: run_command(cmd) run_command(self.cmd) diff --git a\/waflib\/Task.py b\/waflib\/Task.py index 6aebc6074..cb49a7394 100644 --- a\/waflib\/Task.py +++ b\/waflib\/Task.py @@ -163,10 +163,10 @@ class Task(evil): """File extensions that objects of this task class may create""" before = [] - """List of task class names to execute before instances of this class""" + """The instances of this class are executed before the instances of classes whose names are in this list""" after = [] - """List of task class names to execute after instances of this class""" + """The instances of this class are executed after the instances of classes whose names are in this list""" hcode = Utils.SIG_NIL """String representing an additional hash for the class representation""" @@ -306,25 +306,31 @@ def exec_command(self, cmd, **kw): if hasattr(self, 'stderr'): kw['stderr'] = self.stderr - # workaround for command line length limit: - # http:\/\/support.microsoft.com\/kb\/830473 - if not isinstance(cmd, str) and (len(repr(cmd)) >= 8192 if Utils.is_win32 else len(cmd) > 200000): - cmd, args = self.split_argfile(cmd) - try: - (fd, tmp) = tempfile.mkstemp() - os.write(fd, '\r\n'.join(args).encode()) - os.close(fd) - if Logs.verbose: - Logs.debug('argfile: @%r -> %r', tmp, args) - return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw) - finally: + if not isinstance(cmd, str): + if Utils.is_win32: + # win32 compares the resulting length http:\/\/support.microsoft.com\/kb\/830473 + too_long = sum([len(arg) for arg in cmd]) + len(cmd) > 8192 + else: + # non-win32 counts the amount of arguments (200k) + too_long = len(cmd) > 200000 + + if too_long and getattr(self, 'allow_argsfile', True): + # Shunt arguments to a temporary file if the command is too long. + cmd, args = self.split_argfile(cmd) try: - os.remove(tmp) - except OSError: - # anti-virus and indexers can keep files open -_- - pass - else: - return self.generator.bld.exec_command(cmd, **kw) + (fd, tmp) = tempfile.mkstemp() + os.write(fd, '\r\n'.join(args).encode()) + os.close(fd) + if Logs.verbose: + Logs.debug('argfile: @%r -> %r', tmp, args) + return self.generator.bld.exec_command(cmd + ['@' + tmp], **kw) + finally: + try: + os.remove(tmp) + except OSError: + # anti-virus and indexers can keep files open -_- + pass + return self.generator.bld.exec_command(cmd, **kw) def process(self): """ @@ -1044,7 +1050,7 @@ def funex(c): exec(c, dc) return dc['f'] -re_cond = re.compile('(?P\w+)|(?P\|)|(?P&)') +re_cond = re.compile(r'(?P\w+)|(?P\|)|(?P&)') re_novar = re.compile(r'^(SRC|TGT)\W+.*?$') reg_act = re.compile(r'(?P\\)|(?P\$\$)|(?P\$\{(?P\w+)(?P.*?)\})', re.M) def compile_fun_shell(line): diff --git a\/waflib\/TaskGen.py b\/waflib\/TaskGen.py index a74e6431d..32468f03d 100644 --- a\/waflib\/TaskGen.py +++ b\/waflib\/TaskGen.py @@ -74,7 +74,7 @@ def __init__(self, *k, **kw): else: self.bld = kw['bld'] self.env = self.bld.env.derive() - self.path = self.bld.path # emulate chdir when reading scripts + self.path = kw.get('path', self.bld.path) # by default, emulate chdir when reading scripts # Provide a unique index per folder # This is part of a measure to prevent output file name collisions @@ -400,7 +400,7 @@ def feature(*k): Decorator that registers a task generator method that will be executed when the object attribute ``feature`` contains the corresponding key(s):: - from waflib.Task import feature + from waflib.TaskGen import feature @feature('myfeature') def myfunction(self): print('that is my feature!') @@ -631,12 +631,8 @@ def chmod_fun(tsk): cls.scan = self.scan elif has_deps: def scan(self): - nodes = [] - for x in self.generator.to_list(getattr(self.generator, 'deps', None)): - node = self.generator.path.find_resource(x) - if not node: - self.generator.bld.fatal('Could not find %r (was it declared?)' % x) - nodes.append(node) + deps = getattr(self.generator, 'deps', None) + nodes = self.generator.to_nodes(deps) return [nodes, []] cls.scan = scan @@ -727,7 +723,7 @@ def sequence_order(self): self.bld.prev = self -re_m4 = re.compile('@(\w+)@', re.M) +re_m4 = re.compile(r'@(\w+)@', re.M) class subst_pc(Task.Task): """ @@ -905,7 +901,7 @@ def build(bld): # paranoid safety measure for the general case foo.in->foo.h with ambiguous dependencies for xt in HEADER_EXTS: if b.name.endswith(xt): - tsk.ext_in = tsk.ext_in + ['.h'] + tsk.ext_out = tsk.ext_out + ['.h'] break inst_to = getattr(self, 'install_path', None) diff --git a\/waflib\/Tools\/c_aliases.py b\/waflib\/Tools\/c_aliases.py index c9d53692e..928cfe29c 100644 --- a\/waflib\/Tools\/c_aliases.py +++ b\/waflib\/Tools\/c_aliases.py @@ -38,7 +38,7 @@ def sniff_features(**kw): :return: the list of features for a task generator processing the source files :rtype: list of string """ - exts = get_extensions(kw['source']) + exts = get_extensions(kw.get('source', [])) typ = kw['typ'] feats = [] @@ -47,10 +47,12 @@ def sniff_features(**kw): if x in exts: feats.append('cxx') break - if 'c' in exts or 'vala' in exts or 'gs' in exts: feats.append('c') + if 's' in exts or 'S' in exts: + feats.append('asm') + for x in 'f f90 F F90 for FOR'.split(): if x in exts: feats.append('fc') @@ -66,11 +68,11 @@ def sniff_features(**kw): if typ in ('program', 'shlib', 'stlib'): will_link = False for x in feats: - if x in ('cxx', 'd', 'fc', 'c'): + if x in ('cxx', 'd', 'fc', 'c', 'asm'): feats.append(x + typ) will_link = True if not will_link and not kw.get('features', []): - raise Errors.WafError('Cannot link from %r, try passing eg: features="c cprogram"?' % kw) + raise Errors.WafError('Unable to determine how to link %r, try adding eg: features="c cshlib"?' % kw) return feats def set_features(kw, typ): diff --git a\/waflib\/Tools\/c_config.py b\/waflib\/Tools\/c_config.py index d2b3c0d8f..f5ab19bf6 100644 --- a\/waflib\/Tools\/c_config.py +++ b\/waflib\/Tools\/c_config.py @@ -68,6 +68,8 @@ '__s390__' : 's390', '__sh__' : 'sh', '__xtensa__' : 'xtensa', +'__e2k__' : 'e2k', +'__riscv' : 'riscv', } @conf @@ -86,6 +88,10 @@ def configure(conf): :type uselib_store: string :param env: config set or conf.env by default :type env: :py:class:`waflib.ConfigSet.ConfigSet` + :param force_static: force usage of static libraries + :type force_static: bool default False + :param posix: usage of POSIX mode for shlex lexical analiysis library + :type posix: bool default True """ assert(isinstance(line, str)) @@ -103,6 +109,8 @@ def configure(conf): lex.commenters = '' lst = list(lex) + so_re = re.compile(r"\.so(?:\.[0-9]+)*$") + # append_unique is not always possible # for example, apple flags may require both -arch i386 and -arch ppc uselib = uselib_store @@ -144,7 +152,7 @@ def appu(var, val): elif x.startswith('-std='): prefix = 'CXXFLAGS' if '++' in x else 'CFLAGS' app(prefix, x) - elif x.startswith('+') or x in ('-pthread', '-fPIC', '-fpic', '-fPIE', '-fpie'): + elif x.startswith('+') or x in ('-pthread', '-fPIC', '-fpic', '-fPIE', '-fpie', '-flto', '-fno-lto'): app('CFLAGS', x) app('CXXFLAGS', x) app('LINKFLAGS', x) @@ -180,7 +188,7 @@ def appu(var, val): app('CFLAGS', tmp) app('CXXFLAGS', tmp) app('LINKFLAGS', tmp) - elif x.endswith(('.a', '.so', '.dylib', '.lib')): + elif x.endswith(('.a', '.dylib', '.lib')) or so_re.search(x): appu('LINKFLAGS', x) # not cool, #762 else: self.to_log('Unhandled flag %r' % x) @@ -246,13 +254,15 @@ def exec_cfg(self, kw): * if modversion is given, then return the module version * else, execute the *-config* program with the *args* and *variables* given, and set the flags on the *conf.env.FLAGS_name* variable + :param path: the **-config program to use** + :type path: list of string :param atleast_pkgconfig_version: minimum pkg-config version to use (disable other tests) :type atleast_pkgconfig_version: string :param package: package name, for example *gtk+-2.0* :type package: string - :param uselib_store: if the test is successful, define HAVE\_*name*. It is also used to define *conf.env.FLAGS_name* variables. + :param uselib_store: if the test is successful, define HAVE\\_*name*. It is also used to define *conf.env.FLAGS_name* variables. :type uselib_store: string - :param modversion: if provided, return the version of the given module and define *name*\_VERSION + :param modversion: if provided, return the version of the given module and define *name*\\_VERSION :type modversion: string :param args: arguments to give to *package* when retrieving flags :type args: list of string @@ -260,6 +270,12 @@ def exec_cfg(self, kw): :type variables: list of string :param define_variable: additional variables to define (also in conf.env.PKG_CONFIG_DEFINES) :type define_variable: dict(string: string) + :param pkg_config_path: paths where pkg-config should search for .pc config files (overrides env.PKG_CONFIG_PATH if exists) + :type pkg_config_path: string, list of directories separated by colon + :param force_static: force usage of static libraries + :type force_static: bool default False + :param posix: usage of POSIX mode for shlex lexical analiysis library + :type posix: bool default True """ path = Utils.to_list(kw['path']) @@ -334,6 +350,7 @@ def check_cfg(self, *k, **kw): """ Checks for configuration flags using a **-config**-like program (pkg-config, sdl-config, etc). This wraps internal calls to :py:func:`waflib.Tools.c_config.validate_cfg` and :py:func:`waflib.Tools.c_config.exec_cfg` + so check exec_cfg parameters descriptions for more details on kw passed A few examples:: @@ -659,20 +676,21 @@ class test_exec(Task.Task): """ color = 'PINK' def run(self): + cmd = [self.inputs[0].abspath()] + getattr(self.generator, 'test_args', []) if getattr(self.generator, 'rpath', None): if getattr(self.generator, 'define_ret', False): - self.generator.bld.retval = self.generator.bld.cmd_and_log([self.inputs[0].abspath()]) + self.generator.bld.retval = self.generator.bld.cmd_and_log(cmd) else: - self.generator.bld.retval = self.generator.bld.exec_command([self.inputs[0].abspath()]) + self.generator.bld.retval = self.generator.bld.exec_command(cmd) else: env = self.env.env or {} env.update(dict(os.environ)) for var in ('LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'PATH'): env[var] = self.inputs[0].parent.abspath() + os.path.pathsep + env.get(var, '') if getattr(self.generator, 'define_ret', False): - self.generator.bld.retval = self.generator.bld.cmd_and_log([self.inputs[0].abspath()], env=env) + self.generator.bld.retval = self.generator.bld.cmd_and_log(cmd, env=env) else: - self.generator.bld.retval = self.generator.bld.exec_command([self.inputs[0].abspath()], env=env) + self.generator.bld.retval = self.generator.bld.exec_command(cmd, env=env) @feature('test_exec') @after_method('apply_link') @@ -1266,10 +1284,11 @@ def to_log(self, *k, **kw): tasks = [] id_to_task = {} - for dct in k: + for counter, dct in enumerate(k): x = Task.classes['cfgtask'](bld=bld, env=None) tasks.append(x) x.args = dct + x.args['multicheck_counter'] = counter x.bld = bld x.conf = self x.args = dct diff --git a\/waflib\/Tools\/c_preproc.py b\/waflib\/Tools\/c_preproc.py index 7e04b4a7c..68e5f5aea 100644 --- a\/waflib\/Tools\/c_preproc.py +++ b\/waflib\/Tools\/c_preproc.py @@ -75,13 +75,13 @@ class PreprocError(Errors.WafError): re.IGNORECASE | re.MULTILINE) """Match #include lines""" -re_mac = re.compile("^[a-zA-Z_]\w*") +re_mac = re.compile(r"^[a-zA-Z_]\w*") """Match macro definitions""" re_fun = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*[(]') """Match macro functions""" -re_pragma_once = re.compile('^\s*once\s*', re.IGNORECASE) +re_pragma_once = re.compile(r'^\s*once\s*', re.IGNORECASE) """Match #pragma once statements""" re_nl = re.compile('\\\\\r*\n', re.MULTILINE) @@ -660,7 +660,7 @@ def extract_macro(txt): # empty define, assign an empty token return (v, [[], [('T','')]]) -re_include = re.compile('^\s*(<(?:.*)>|"(?:.*)")') +re_include = re.compile(r'^\s*(<(?:.*)>|"(?:.*)")') def extract_include(txt, defs): """ Process a line in the form:: diff --git a\/waflib\/Tools\/c_tests.py b\/waflib\/Tools\/c_tests.py index f858df576..bdd186c6b 100644 --- a\/waflib\/Tools\/c_tests.py +++ b\/waflib\/Tools\/c_tests.py @@ -180,9 +180,15 @@ def check_large_file(self, **kw): ######################################################################################## ENDIAN_FRAGMENT = ''' +#ifdef _MSC_VER +#define testshlib_EXPORT __declspec(dllexport) +#else +#define testshlib_EXPORT +#endif + short int ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 }; short int ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 }; -int use_ascii (int i) { +int testshlib_EXPORT use_ascii (int i) { return ascii_mm[i] + ascii_ii[i]; } short int ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 }; @@ -208,12 +214,12 @@ def run(self): return -1 @feature('grep_for_endianness') -@after_method('process_source') +@after_method('apply_link') def grep_for_endianness_fun(self): """ Used by the endianness configuration test """ - self.create_task('grep_for_endianness', self.compiled_tasks[0].outputs[0]) + self.create_task('grep_for_endianness', self.link_task.outputs[0]) @conf def check_endianness(self): @@ -223,7 +229,9 @@ def check_endianness(self): tmp = [] def check_msg(self): return tmp[0] - self.check(fragment=ENDIAN_FRAGMENT, features='c grep_for_endianness', - msg='Checking for endianness', define='ENDIANNESS', tmp=tmp, okmsg=check_msg) + + self.check(fragment=ENDIAN_FRAGMENT, features='c cshlib grep_for_endianness', + msg='Checking for endianness', define='ENDIANNESS', tmp=tmp, + okmsg=check_msg, confcache=None) return tmp[0] diff --git a\/waflib\/Tools\/ccroot.py b\/waflib\/Tools\/ccroot.py index 484846f5f..533992903 100644 --- a\/waflib\/Tools\/ccroot.py +++ b\/waflib\/Tools\/ccroot.py @@ -111,7 +111,7 @@ def apply_incpaths(self): tg = bld(features='includes', includes='.') The folders only need to be relative to the current directory, the equivalent build directory is - added automatically (for headers created in the build directory). This enable using a build directory + added automatically (for headers created in the build directory). This enables using a build directory or not (``top == out``). This method will add a list of nodes read by :py:func:`waflib.Tools.ccroot.to_incnodes` in ``tg.env.INCPATHS``, @@ -128,6 +128,7 @@ class link_task(Task.Task): Base class for all link tasks. A task generator is supposed to have at most one link task bound in the attribute *link_task*. See :py:func:`waflib.Tools.ccroot.apply_link`. .. inheritance-diagram:: waflib.Tools.ccroot.stlink_task waflib.Tools.c.cprogram waflib.Tools.c.cshlib waflib.Tools.cxx.cxxstlib waflib.Tools.cxx.cxxprogram waflib.Tools.cxx.cxxshlib waflib.Tools.d.dprogram waflib.Tools.d.dshlib waflib.Tools.d.dstlib waflib.Tools.ccroot.fake_shlib waflib.Tools.ccroot.fake_stlib waflib.Tools.asm.asmprogram waflib.Tools.asm.asmshlib waflib.Tools.asm.asmstlib + :top-classes: waflib.Tools.ccroot.link_task """ color = 'YELLOW' @@ -238,6 +239,17 @@ def wrap(self): setattr(cls, 'run', wrap) rm_tgt(stlink_task) +@feature('skip_stlib_link_deps') +@before_method('process_use') +def apply_skip_stlib_link_deps(self): + """ + This enables an optimization in the :py:func:wafilb.Tools.ccroot.processes_use: method that skips dependency and + link flag optimizations for targets that generate static libraries (via the :py:class:Tools.ccroot.stlink_task task). + The actual behavior is implemented in :py:func:wafilb.Tools.ccroot.processes_use: method so this feature only tells waf + to enable the new behavior. + """ + self.env.SKIP_STLIB_LINK_DEPS = True + @feature('c', 'cxx', 'd', 'fc', 'asm') @after_method('process_source') def apply_link(self): @@ -386,7 +398,11 @@ def build(bld): y = self.bld.get_tgen_by_name(x) var = y.tmp_use_var if var and link_task: - if var == 'LIB' or y.tmp_use_stlib or x in names: + if self.env.SKIP_STLIB_LINK_DEPS and isinstance(link_task, stlink_task): + # If the skip_stlib_link_deps feature is enabled then we should + # avoid adding lib deps to the stlink_task instance. + pass + elif var == 'LIB' or y.tmp_use_stlib or x in names: self.env.append_value(var, [y.target[y.target.rfind(os.sep) + 1:]]) self.link_task.dep_nodes.extend(y.link_task.outputs) tmp_path = y.link_task.outputs[0].parent.path_from(self.get_cwd()) diff --git a\/waflib\/Tools\/compiler_c.py b\/waflib\/Tools\/compiler_c.py index 2dba3f827..e033ce6c5 100644 --- a\/waflib\/Tools\/compiler_c.py +++ b\/waflib\/Tools\/compiler_c.py @@ -36,18 +36,19 @@ def build(bld): from waflib.Logs import debug c_compiler = { -'win32': ['msvc', 'gcc', 'clang'], -'cygwin': ['gcc'], -'darwin': ['clang', 'gcc'], -'aix': ['xlc', 'gcc', 'clang'], -'linux': ['gcc', 'clang', 'icc'], -'sunos': ['suncc', 'gcc'], -'irix': ['gcc', 'irixcc'], -'hpux': ['gcc'], -'osf1V': ['gcc'], -'gnu': ['gcc', 'clang'], -'java': ['gcc', 'msvc', 'clang', 'icc'], -'default':['clang', 'gcc'], +'win32': ['msvc', 'gcc', 'clang'], +'cygwin': ['gcc', 'clang'], +'darwin': ['clang', 'gcc'], +'aix': ['xlc', 'gcc', 'clang'], +'linux': ['gcc', 'clang', 'icc'], +'sunos': ['suncc', 'gcc'], +'irix': ['gcc', 'irixcc'], +'hpux': ['gcc'], +'osf1V': ['gcc'], +'gnu': ['gcc', 'clang'], +'java': ['gcc', 'msvc', 'clang', 'icc'], +'gnukfreebsd': ['gcc', 'clang'], +'default': ['clang', 'gcc'], } """ Dict mapping platform names to Waf tools finding specific C compilers:: diff --git a\/waflib\/Tools\/compiler_cxx.py b\/waflib\/Tools\/compiler_cxx.py index 1af65a226..42658c584 100644 --- a\/waflib\/Tools\/compiler_cxx.py +++ b\/waflib\/Tools\/compiler_cxx.py @@ -37,18 +37,19 @@ def build(bld): from waflib.Logs import debug cxx_compiler = { -'win32': ['msvc', 'g++', 'clang++'], -'cygwin': ['g++'], -'darwin': ['clang++', 'g++'], -'aix': ['xlc++', 'g++', 'clang++'], -'linux': ['g++', 'clang++', 'icpc'], -'sunos': ['sunc++', 'g++'], -'irix': ['g++'], -'hpux': ['g++'], -'osf1V': ['g++'], -'gnu': ['g++', 'clang++'], -'java': ['g++', 'msvc', 'clang++', 'icpc'], -'default': ['clang++', 'g++'] +'win32': ['msvc', 'g++', 'clang++'], +'cygwin': ['g++', 'clang++'], +'darwin': ['clang++', 'g++'], +'aix': ['xlc++', 'g++', 'clang++'], +'linux': ['g++', 'clang++', 'icpc'], +'sunos': ['sunc++', 'g++'], +'irix': ['g++'], +'hpux': ['g++'], +'osf1V': ['g++'], +'gnu': ['g++', 'clang++'], +'java': ['g++', 'msvc', 'clang++', 'icpc'], +'gnukfreebsd': ['g++', 'clang++'], +'default': ['clang++', 'g++'] } """ Dict mapping the platform names to Waf tools finding specific C++ compilers:: diff --git a\/waflib\/Tools\/irixcc.py b\/waflib\/Tools\/irixcc.py index c3ae1ac91..0335c13cb 100644 --- a\/waflib\/Tools\/irixcc.py +++ b\/waflib\/Tools\/irixcc.py @@ -13,22 +13,11 @@ @conf def find_irixcc(conf): v = conf.env - cc = None - if v.CC: - cc = v.CC - elif 'CC' in conf.environ: - cc = conf.environ['CC'] - if not cc: - cc = conf.find_program('cc', var='CC') - if not cc: - conf.fatal('irixcc was not found') - + cc = conf.find_program('cc', var='CC') try: conf.cmd_and_log(cc + ['-version']) except Errors.WafError: conf.fatal('%r -version could not be executed' % cc) - - v.CC = cc v.CC_NAME = 'irix' @conf @@ -57,7 +46,6 @@ def irixcc_common_flags(conf): def configure(conf): conf.find_irixcc() - conf.find_cpp() conf.find_ar() conf.irixcc_common_flags() conf.cc_load_tools() diff --git a\/waflib\/Tools\/msvc.py b\/waflib\/Tools\/msvc.py index 17b347d45..d60f67026 100644 --- a\/waflib\/Tools\/msvc.py +++ b\/waflib\/Tools\/msvc.py @@ -99,10 +99,31 @@ def build(bld): """List of icl platforms""" def options(opt): - opt.add_option('--msvc_version', type='string', help = 'msvc version, eg: "msvc 10.0,msvc 9.0"', default='') + default_ver = '' + vsver = os.getenv('VSCMD_VER') + if vsver: + m = re.match(r'(^\d+\.\d+).*', vsver) + if m: + default_ver = 'msvc %s' % m.group(1) + opt.add_option('--msvc_version', type='string', help = 'msvc version, eg: "msvc 10.0,msvc 9.0"', default=default_ver) opt.add_option('--msvc_targets', type='string', help = 'msvc targets, eg: "x64,arm"', default='') opt.add_option('--no-msvc-lazy', action='store_false', help = 'lazily check msvc target environments', default=True, dest='msvc_lazy') +class MSVCVersion(object): + def __init__(self, ver): + m = re.search(r'^(.*)\s+(\d+[.]\d+)', ver) + if m: + self.name = m.group(1) + self.number = float(m.group(2)) + else: + self.name = ver + self.number = 0. + + def __lt__(self, other): + if self.number == other.number: + return self.name < other.name + return self.number < other.number + @conf def setup_msvc(conf, versiondict): """ @@ -119,7 +140,7 @@ def setup_msvc(conf, versiondict): platforms=Utils.to_list(conf.env.MSVC_TARGETS) or [i for i,j in all_msvc_platforms+all_icl_platforms+all_wince_platforms] desired_versions = getattr(Options.options, 'msvc_version', '').split(',') if desired_versions == ['']: - desired_versions = conf.env.MSVC_VERSIONS or list(reversed(sorted(versiondict.keys()))) + desired_versions = conf.env.MSVC_VERSIONS or list(sorted(versiondict.keys(), key=MSVCVersion, reverse=True)) # Override lazy detection by evaluating after the fact. lazy_detect = getattr(Options.options, 'msvc_lazy', True) @@ -187,7 +208,7 @@ def get_msvc_version(conf, compiler, version, target, vcvars): echo INCLUDE=%%INCLUDE%% echo LIB=%%LIB%%;%%LIBPATH%% """ % (vcvars,target)) - sout = conf.cmd_and_log(['cmd.exe', '\/E:on', '\/V:on', '\/C', batfile.abspath()]) + sout = conf.cmd_and_log(['cmd.exe', '\/E:on', '\/V:on', '\/C', batfile.abspath()], stdin=getattr(Utils.subprocess, 'DEVNULL', None)) lines = sout.splitlines() if not lines[0]: @@ -281,7 +302,7 @@ def gather_wince_supported_platforms(): def gather_msvc_detected_versions(): #Detected MSVC versions! - version_pattern = re.compile('^(\d\d?\.\d\d?)(Exp)?$') + version_pattern = re.compile(r'^(\d\d?\.\d\d?)(Exp)?$') detected_versions = [] for vcver,vcvar in (('VCExpress','Exp'), ('VisualStudio','')): prefix = 'SOFTWARE\\Wow6432node\\Microsoft\\' + vcver @@ -367,7 +388,7 @@ def gather_wsdk_versions(conf, versions): :param versions: list to modify :type versions: list """ - version_pattern = re.compile('^v..?.?\...?.?') + version_pattern = re.compile(r'^v..?.?\...?.?') try: all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Microsoft\\Microsoft SDKs\\Windows') except OSError: @@ -525,7 +546,7 @@ def gather_icl_versions(conf, versions): :param versions: list to modify :type versions: list """ - version_pattern = re.compile('^...?.?\....?.?') + version_pattern = re.compile(r'^...?.?\....?.?') try: all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Intel\\Compilers\\C++') except OSError: @@ -579,7 +600,7 @@ def gather_intel_composer_versions(conf, versions): :param versions: list to modify :type versions: list """ - version_pattern = re.compile('^...?.?\...?.?.?') + version_pattern = re.compile(r'^...?.?\...?.?.?') try: all_versions = Utils.winreg.OpenKey(Utils.winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Wow6432node\\Intel\\Suites') except OSError: @@ -683,7 +704,7 @@ def find_lt_names_msvc(self, libname, is_static=False): if not is_static and ltdict.get('library_names', ''): dllnames=ltdict['library_names'].split() dll=dllnames[0].lower() - dll=re.sub('\.dll$', '', dll) + dll=re.sub(r'\.dll$', '', dll) return (lt_libdir, dll, False) elif ltdict.get('old_library', ''): olib=ltdict['old_library'] @@ -700,7 +721,7 @@ def find_lt_names_msvc(self, libname, is_static=False): @conf def libname_msvc(self, libname, is_static=False): lib = libname.lower() - lib = re.sub('\.lib$','',lib) + lib = re.sub(r'\.lib$','',lib) if lib in g_msvc_systemlibs: return lib @@ -747,11 +768,11 @@ def libname_msvc(self, libname, is_static=False): for libn in libnames: if os.path.exists(os.path.join(path, libn)): Logs.debug('msvc: lib found: %s', os.path.join(path,libn)) - return re.sub('\.lib$', '',libn) + return re.sub(r'\.lib$', '',libn) #if no lib can be found, just return the libname as msvc expects it self.fatal('The library %r could not be found' % libname) - return re.sub('\.lib$', '', libname) + return re.sub(r'\.lib$', '', libname) @conf def check_lib_msvc(self, libname, is_static=False, uselib_store=None): @@ -969,7 +990,7 @@ def build(bld): if not is_static: for f in self.env.LINKFLAGS: d = f.lower() - if d[1:] == 'debug': + if d[1:] in ('debug', 'debug:full', 'debug:fastlink'): pdbnode = self.link_task.outputs[0].change_ext('.pdb') self.link_task.outputs.append(pdbnode) diff --git a\/waflib\/Tools\/waf_unit_test.py b\/waflib\/Tools\/waf_unit_test.py index 74d6c0561..8cff89bde 100644 --- a\/waflib\/Tools\/waf_unit_test.py +++ b\/waflib\/Tools\/waf_unit_test.py @@ -97,6 +97,7 @@ def make_interpreted_test(self): if isinstance(v, str): v = v.split(os.pathsep) self.ut_env[k] = os.pathsep.join(p + v) + self.env.append_value('UT_DEPS', ['%r%r' % (key, self.ut_env[key]) for key in self.ut_env]) @feature('test') @after_method('apply_link', 'process_use') @@ -108,7 +109,8 @@ def make_test(self): tsk = self.create_task('utest', self.link_task.outputs) if getattr(self, 'ut_str', None): self.ut_run, lst = Task.compile_fun(self.ut_str, shell=getattr(self, 'ut_shell', False)) - tsk.vars = lst + tsk.vars + tsk.vars = tsk.vars + lst + self.env.append_value('UT_DEPS', self.ut_str) self.handle_ut_cwd('ut_cwd') @@ -139,6 +141,10 @@ def add_path(var): if not hasattr(self, 'ut_cmd'): self.ut_cmd = getattr(Options.options, 'testcmd', False) + self.env.append_value('UT_DEPS', str(self.ut_cmd)) + self.env.append_value('UT_DEPS', self.ut_paths) + self.env.append_value('UT_DEPS', ['%r%r' % (key, self.ut_env[key]) for key in self.ut_env]) + @taskgen_method def add_test_results(self, tup): """Override and return tup[1] to interrupt the build immediately if a test does not run""" @@ -159,7 +165,7 @@ class utest(Task.Task): """ color = 'PINK' after = ['vnum', 'inst'] - vars = [] + vars = ['UT_DEPS'] def runnable_status(self): """ @@ -200,7 +206,7 @@ def run(self): self.ut_exec = getattr(self.generator, 'ut_exec', [self.inputs[0].abspath()]) ut_cmd = getattr(self.generator, 'ut_cmd', False) if ut_cmd: - self.ut_exec = shlex.split(ut_cmd % ' '.join(self.ut_exec)) + self.ut_exec = shlex.split(ut_cmd % Utils.shell_escape(self.ut_exec)) return self.exec_command(self.ut_exec) @@ -214,7 +220,7 @@ def exec_command(self, cmd, **kw): 'cmd': cmd } script_file = self.inputs[0].abspath() + '_run.py' - Utils.writef(script_file, script_code) + Utils.writef(script_file, script_code, encoding='utf-8') os.chmod(script_file, Utils.O755) if Logs.verbose > 1: Logs.info('Test debug file written as %r' % script_file) diff --git a\/waflib\/Utils.py b\/waflib\/Utils.py index a0cc2a09d..ea0f7a9db 100644 --- a\/waflib\/Utils.py +++ b\/waflib\/Utils.py @@ -11,7 +11,7 @@ from __future__ import with_statement -import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time +import atexit, os, sys, errno, inspect, re, datetime, platform, base64, signal, functools, time, shlex try: import cPickle @@ -49,10 +49,16 @@ class TimeoutExpired(Exception): from hashlib import md5 except ImportError: try: - from md5 import md5 + from hashlib import sha1 as md5 except ImportError: - # never fail to enable fixes from another module + # never fail to enable potential fixes from another module pass +else: + try: + md5().digest() + except ValueError: + # Fips? #2213 + from hashlib import sha1 as md5 try: import threading @@ -202,7 +208,7 @@ def __next__(self): next = __next__ -is_win32 = os.sep == '\\' or sys.platform == 'win32' # msys2 +is_win32 = os.sep == '\\' or sys.platform == 'win32' or os.name == 'nt' # msys2 """ Whether this system is a Windows series """ @@ -446,6 +452,8 @@ def console_encoding(): pass else: if codepage: + if 65001 == codepage and sys.version_info < (3, 3): + return 'utf-8' return 'cp%d' % codepage return sys.stdout.encoding or ('cp1252' if is_win32 else 'latin-1') @@ -484,7 +492,9 @@ def split_path_msys(path): if sys.platform == 'cygwin': split_path = split_path_cygwin elif is_win32: - if os.environ.get('MSYSTEM'): + # Consider this an MSYSTEM environment if $MSYSTEM is set and python + # reports is executable from a unix like path on a windows host. + if os.environ.get('MSYSTEM') and sys.executable.startswith('\/'): split_path = split_path_msys else: split_path = split_path_win32 @@ -569,10 +579,13 @@ def quote_define_name(s): fu = fu.upper() return fu -re_sh = re.compile('\\s|\'|"') -""" -Regexp used for shell_escape below -""" +# shlex.quote didn't exist until python 3.3. Prior to that it was a non-documented +# function in pipes. +try: + shell_quote = shlex.quote +except AttributeError: + import pipes + shell_quote = pipes.quote def shell_escape(cmd): """ @@ -581,7 +594,7 @@ def shell_escape(cmd): """ if isinstance(cmd, str): return cmd - return ' '.join(repr(x) if re_sh.search(x) else x for x in cmd) + return ' '.join(shell_quote(x) for x in cmd) def h_list(lst): """ @@ -596,6 +609,12 @@ def h_list(lst): """ return md5(repr(lst).encode()).digest() +if sys.hexversion < 0x3000000: + def h_list_python2(lst): + return md5(repr(lst)).digest() + h_list_python2.__doc__ = h_list.__doc__ + h_list = h_list_python2 + def h_fun(fun): """ Hash functions @@ -615,7 +634,7 @@ def h_fun(fun): # # The sorting result outcome will be consistent because: # 1. tuples are compared in order of their elements - # 2. optional argument names are unique + # 2. optional argument namess are unique code.extend(sorted(fun.keywords.items())) code.append(h_fun(fun.func)) fun.code = h_list(code) @@ -730,7 +749,7 @@ def unversioned_sys_platform(): if s == 'cli' and os.name == 'nt': # ironpython is only on windows as far as we know return 'win32' - return re.split('\d+$', s)[0] + return re.split(r'\d+$', s)[0] def nada(*k, **kw): """ @@ -851,6 +870,19 @@ def lib64(): return '64' return '' +def loose_version(ver_str): + # private for the time being! + # see #2402 + lst = re.split(r'([.]|\\d+|[a-zA-Z])', ver_str) + ver = [] + for i, val in enumerate(lst): + try: + ver.append(int(val)) + except ValueError: + if val != '.': + ver.append(val) + return ver + def sane_path(p): # private function for the time being! return os.path.abspath(os.path.expanduser(p)) @@ -871,13 +903,13 @@ def get_process(): except IndexError: filepath = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'processor.py' cmd = [sys.executable, '-c', readf(filepath)] - return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0, close_fds=not is_win32) def run_prefork_process(cmd, kwargs, cargs): """ Delegates process execution to a pre-forked process instance. """ - if not 'env' in kwargs: + if not kwargs.get('env'): kwargs['env'] = dict(os.environ) try: obj = base64.b64encode(cPickle.dumps([cmd, kwargs, cargs])) diff --git a\/waflib\/ansiterm.py b\/waflib\/ansiterm.py index 0d20c6374..027f0ad68 100644 --- a\/waflib\/ansiterm.py +++ b\/waflib\/ansiterm.py @@ -264,7 +264,7 @@ def hide_cursor(self,param): 'u': pop_cursor, } # Match either the escape sequence or text not containing escape sequence - ansi_tokens = re.compile('(?:\x1b\[([0-9?;]*)([a-zA-Z])|([^\x1b]+))') + ansi_tokens = re.compile(r'(?:\x1b\[([0-9?;]*)([a-zA-Z])|([^\x1b]+))') def write(self, text): try: wlock.acquire() diff --git a\/waflib\/extras\/clang_cross.py b\/waflib\/extras\/clang_cross.py new file mode 100644 index 000000000..1b51e2886 --- \/dev\/null +++ b\/waflib\/extras\/clang_cross.py @@ -0,0 +1,92 @@ +#!\/usr\/bin\/env python +# encoding: utf-8 +# Krzysztof Kosiński 2014 +# DragoonX6 2018 + +""" +Detect the Clang C compiler +This version is an attempt at supporting the -target and -sysroot flag of Clang. +""" + +from waflib.Tools import ccroot, ar, gcc +from waflib.Configure import conf +import waflib.Context +import waflib.extras.clang_cross_common + +def options(opt): + """ + Target triplet for clang:: + $ waf configure --clang-target-triple=x86_64-pc-linux-gnu + """ + cc_compiler_opts = opt.add_option_group('Configuration options') + cc_compiler_opts.add_option('--clang-target-triple', default=None, + help='Target triple for clang', + dest='clang_target_triple') + cc_compiler_opts.add_option('--clang-sysroot', default=None, + help='Sysroot for clang', + dest='clang_sysroot') + +@conf +def find_clang(conf): + """ + Finds the program clang and executes it to ensure it really is clang + """ + + import os + + cc = conf.find_program('clang', var='CC') + + if conf.options.clang_target_triple != None: + conf.env.append_value('CC', ['-target', conf.options.clang_target_triple]) + + if conf.options.clang_sysroot != None: + sysroot = str() + + if os.path.isabs(conf.options.clang_sysroot): + sysroot = conf.options.clang_sysroot + else: + sysroot = os.path.normpath(os.path.join(os.getcwd(), conf.options.clang_sysroot)) + + conf.env.append_value('CC', ['--sysroot', sysroot]) + + conf.get_cc_version(cc, clang=True) + conf.env.CC_NAME = 'clang' + +@conf +def clang_modifier_x86_64_w64_mingw32(conf): + conf.gcc_modifier_win32() + +@conf +def clang_modifier_i386_w64_mingw32(conf): + conf.gcc_modifier_win32() + +@conf +def clang_modifier_x86_64_windows_msvc(conf): + conf.clang_modifier_msvc() + + # Allow the user to override any flags if they so desire. + clang_modifier_user_func = getattr(conf, 'clang_modifier_x86_64_windows_msvc_user', None) + if clang_modifier_user_func: + clang_modifier_user_func() + +@conf +def clang_modifier_i386_windows_msvc(conf): + conf.clang_modifier_msvc() + + # Allow the user to override any flags if they so desire. + clang_modifier_user_func = getattr(conf, 'clang_modifier_i386_windows_msvc_user', None) + if clang_modifier_user_func: + clang_modifier_user_func() + +def configure(conf): + conf.find_clang() + conf.find_program(['llvm-ar', 'ar'], var='AR') + conf.find_ar() + conf.gcc_common_flags() + # Allow the user to provide flags for the target platform. + conf.gcc_modifier_platform() + # And allow more fine grained control based on the compiler's triplet. + conf.clang_modifier_target_triple() + conf.cc_load_tools() + conf.cc_add_flags() + conf.link_add_flags() diff --git a\/waflib\/extras\/clang_cross_common.py b\/waflib\/extras\/clang_cross_common.py new file mode 100644 index 000000000..b76a07006 --- \/dev\/null +++ b\/waflib\/extras\/clang_cross_common.py @@ -0,0 +1,113 @@ +#!\/usr\/bin\/env python +# encoding: utf-8 +# DragoonX6 2018 + +""" +Common routines for cross_clang.py and cross_clangxx.py +""" + +from waflib.Configure import conf +import waflib.Context + +def normalize_target_triple(target_triple): + target_triple = target_triple[:-1] + normalized_triple = target_triple.replace('--', '-unknown-') + + if normalized_triple.startswith('-'): + normalized_triple = 'unknown' + normalized_triple + + if normalized_triple.endswith('-'): + normalized_triple += 'unknown' + + # Normalize MinGW builds to *arch*-w64-mingw32 + if normalized_triple.endswith('windows-gnu'): + normalized_triple = normalized_triple[:normalized_triple.index('-')] + '-w64-mingw32' + + # Strip the vendor when doing msvc builds, since it's unused anyway. + if normalized_triple.endswith('windows-msvc'): + normalized_triple = normalized_triple[:normalized_triple.index('-')] + '-windows-msvc' + + return normalized_triple.replace('-', '_') + +@conf +def clang_modifier_msvc(conf): + import os + + """ + Really basic setup to use clang in msvc mode. + We actually don't really want to do a lot, even though clang is msvc compatible + in this mode, that doesn't mean we're actually using msvc. + It's probably the best to leave it to the user, we can assume msvc mode if the user + uses the clang-cl frontend, but this module only concerns itself with the gcc-like frontend. + """ + v = conf.env + v.cprogram_PATTERN = '%s.exe' + + v.cshlib_PATTERN = '%s.dll' + v.implib_PATTERN = '%s.lib' + v.IMPLIB_ST = '-Wl,-IMPLIB:%s' + v.SHLIB_MARKER = [] + + v.CFLAGS_cshlib = [] + v.LINKFLAGS_cshlib = ['-Wl,-DLL'] + v.cstlib_PATTERN = '%s.lib' + v.STLIB_MARKER = [] + + del(v.AR) + conf.find_program(['llvm-lib', 'lib'], var='AR') + v.ARFLAGS = ['-nologo'] + v.AR_TGT_F = ['-out:'] + + # Default to the linker supplied with llvm instead of link.exe or ld + v.LINK_CC = v.CC + ['-fuse-ld=lld', '-nostdlib'] + v.CCLNK_TGT_F = ['-o'] + v.def_PATTERN = '-Wl,-def:%s' + + v.LINKFLAGS = [] + + v.LIB_ST = '-l%s' + v.LIBPATH_ST = '-Wl,-LIBPATH:%s' + v.STLIB_ST = '-l%s' + v.STLIBPATH_ST = '-Wl,-LIBPATH:%s' + + CFLAGS_CRT_COMMON = [ + '-Xclang', '--dependent-lib=oldnames', + '-Xclang', '-fno-rtti-data', + '-D_MT' + ] + + v.CFLAGS_CRT_MULTITHREADED = CFLAGS_CRT_COMMON + [ + '-Xclang', '-flto-visibility-public-std', + '-Xclang', '--dependent-lib=libcmt', + ] + v.CXXFLAGS_CRT_MULTITHREADED = v.CFLAGS_CRT_MULTITHREADED + + v.CFLAGS_CRT_MULTITHREADED_DBG = CFLAGS_CRT_COMMON + [ + '-D_DEBUG', + '-Xclang', '-flto-visibility-public-std', + '-Xclang', '--dependent-lib=libcmtd', + ] + v.CXXFLAGS_CRT_MULTITHREADED_DBG = v.CFLAGS_CRT_MULTITHREADED_DBG + + v.CFLAGS_CRT_MULTITHREADED_DLL = CFLAGS_CRT_COMMON + [ + '-D_DLL', + '-Xclang', '--dependent-lib=msvcrt' + ] + v.CXXFLAGS_CRT_MULTITHREADED_DLL = v.CFLAGS_CRT_MULTITHREADED_DLL + + v.CFLAGS_CRT_MULTITHREADED_DLL_DBG = CFLAGS_CRT_COMMON + [ + '-D_DLL', + '-D_DEBUG', + '-Xclang', '--dependent-lib=msvcrtd', + ] + v.CXXFLAGS_CRT_MULTITHREADED_DLL_DBG = v.CFLAGS_CRT_MULTITHREADED_DLL_DBG + +@conf +def clang_modifier_target_triple(conf, cpp=False): + compiler = conf.env.CXX if cpp else conf.env.CC + output = conf.cmd_and_log(compiler + ['-dumpmachine'], output=waflib.Context.STDOUT) + + modifier = ('clangxx' if cpp else 'clang') + '_modifier_' + clang_modifier_func = getattr(conf, modifier + normalize_target_triple(output), None) + if clang_modifier_func: + clang_modifier_func() diff --git a\/waflib\/extras\/clangxx_cross.py b\/waflib\/extras\/clangxx_cross.py new file mode 100644 index 000000000..0ad38ad46 --- \/dev\/null +++ b\/waflib\/extras\/clangxx_cross.py @@ -0,0 +1,106 @@ +#!\/usr\/bin\/env python +# encoding: utf-8 +# Thomas Nagy 2009-2018 (ita) +# DragoonX6 2018 + +""" +Detect the Clang++ C++ compiler +This version is an attempt at supporting the -target and -sysroot flag of Clang++. +""" + +from waflib.Tools import ccroot, ar, gxx +from waflib.Configure import conf +import waflib.extras.clang_cross_common + +def options(opt): + """ + Target triplet for clang++:: + $ waf configure --clangxx-target-triple=x86_64-pc-linux-gnu + """ + cxx_compiler_opts = opt.add_option_group('Configuration options') + cxx_compiler_opts.add_option('--clangxx-target-triple', default=None, + help='Target triple for clang++', + dest='clangxx_target_triple') + cxx_compiler_opts.add_option('--clangxx-sysroot', default=None, + help='Sysroot for clang++', + dest='clangxx_sysroot') + +@conf +def find_clangxx(conf): + """ + Finds the program clang++, and executes it to ensure it really is clang++ + """ + + import os + + cxx = conf.find_program('clang++', var='CXX') + + if conf.options.clangxx_target_triple != None: + conf.env.append_value('CXX', ['-target', conf.options.clangxx_target_triple]) + + if conf.options.clangxx_sysroot != None: + sysroot = str() + + if os.path.isabs(conf.options.clangxx_sysroot): + sysroot = conf.options.clangxx_sysroot + else: + sysroot = os.path.normpath(os.path.join(os.getcwd(), conf.options.clangxx_sysroot)) + + conf.env.append_value('CXX', ['--sysroot', sysroot]) + + conf.get_cc_version(cxx, clang=True) + conf.env.CXX_NAME = 'clang' + +@conf +def clangxx_modifier_x86_64_w64_mingw32(conf): + conf.gcc_modifier_win32() + +@conf +def clangxx_modifier_i386_w64_mingw32(conf): + conf.gcc_modifier_win32() + +@conf +def clangxx_modifier_msvc(conf): + v = conf.env + v.cxxprogram_PATTERN = v.cprogram_PATTERN + v.cxxshlib_PATTERN = v.cshlib_PATTERN + + v.CXXFLAGS_cxxshlib = [] + v.LINKFLAGS_cxxshlib = v.LINKFLAGS_cshlib + v.cxxstlib_PATTERN = v.cstlib_PATTERN + + v.LINK_CXX = v.CXX + ['-fuse-ld=lld', '-nostdlib'] + v.CXXLNK_TGT_F = v.CCLNK_TGT_F + +@conf +def clangxx_modifier_x86_64_windows_msvc(conf): + conf.clang_modifier_msvc() + conf.clangxx_modifier_msvc() + + # Allow the user to override any flags if they so desire. + clang_modifier_user_func = getattr(conf, 'clangxx_modifier_x86_64_windows_msvc_user', None) + if clang_modifier_user_func: + clang_modifier_user_func() + +@conf +def clangxx_modifier_i386_windows_msvc(conf): + conf.clang_modifier_msvc() + conf.clangxx_modifier_msvc() + + # Allow the user to override any flags if they so desire. + clang_modifier_user_func = getattr(conf, 'clangxx_modifier_i386_windows_msvc_user', None) + if clang_modifier_user_func: + clang_modifier_user_func() + +def configure(conf): + conf.find_clangxx() + conf.find_program(['llvm-ar', 'ar'], var='AR') + conf.find_ar() + conf.gxx_common_flags() + # Allow the user to provide flags for the target platform. + conf.gxx_modifier_platform() + # And allow more fine grained control based on the compiler's triplet. + conf.clang_modifier_target_triple(cpp=True) + conf.cxx_load_tools() + conf.cxx_add_flags() + conf.link_add_flags() diff --git a\/waflib\/extras\/classic_runner.py b\/waflib\/extras\/classic_runner.py new file mode 100644 index 000000000..b08c794e8 --- \/dev\/null +++ b\/waflib\/extras\/classic_runner.py @@ -0,0 +1,68 @@ +#!\/usr\/bin\/env python +# encoding: utf-8 +# Thomas Nagy, 2021 (ita) + +from waflib import Utils, Runner + +""" +Re-enable the classic threading system from waf 1.x + +def configure(conf): + conf.load('classic_runner') +""" + +class TaskConsumer(Utils.threading.Thread): + """ + Task consumers belong to a pool of workers + + They wait for tasks in the queue and then use ``task.process(...)`` + """ + def __init__(self, spawner): + Utils.threading.Thread.__init__(self) + """ + Obtain :py:class:`waflib.Task.TaskBase` instances from this queue. + """ + self.spawner = spawner + self.daemon = True + self.start() + + def run(self): + """ + Loop over the tasks to execute + """ + try: + self.loop() + except Exception: + pass + + def loop(self): + """ + Obtain tasks from :py:attr:`waflib.Runner.TaskConsumer.ready` and call + :py:meth:`waflib.Task.TaskBase.process`. If the object is a function, execute it. + """ + master = self.spawner.master + while 1: + if not master.stop: + try: + tsk = master.ready.get() + if tsk: + tsk.log_display(tsk.generator.bld) + master.process_task(tsk) + else: + break + finally: + master.out.put(tsk) + +class Spawner(object): + """ + Daemon thread that consumes tasks from :py:class:`waflib.Runner.Parallel` producer and + spawns a consuming thread :py:class:`waflib.Runner.Consumer` for each + :py:class:`waflib.Task.Task` instance. + """ + def __init__(self, master): + self.master = master + """:py:class:`waflib.Runner.Parallel` producer instance""" + + self.pool = [TaskConsumer(self) for i in range(master.numjobs)] + +Runner.Spawner = Spawner diff --git a\/waflib\/extras\/color_msvc.py b\/waflib\/extras\/color_msvc.py new file mode 100644 index 000000000..60bacb7b2 --- \/dev\/null +++ b\/waflib\/extras\/color_msvc.py @@ -0,0 +1,59 @@ +#!\/usr\/bin\/env python +# encoding: utf-8 + +# Replaces the default formatter by one which understands MSVC output and colorizes it. +# Modified from color_gcc.py + +__author__ = __maintainer__ = "Alibek Omarov " +__copyright__ = "Alibek Omarov, 2019" + +import sys +from waflib import Logs + +class ColorMSVCFormatter(Logs.formatter): + def __init__(self, colors): + self.colors = colors + Logs.formatter.__init__(self) + + def parseMessage(self, line, color): + # Split messaage from 'disk:filepath: type: message' + arr = line.split(':', 3) + if len(arr) < 4: + return line + + colored = self.colors.BOLD + arr[0] + ':' + arr[1] + ':' + self.colors.NORMAL + colored += color + arr[2] + ':' + self.colors.NORMAL + colored += arr[3] + return colored + + def format(self, rec): + frame = sys._getframe() + while frame: + func = frame.f_code.co_name + if func == 'exec_command': + cmd = frame.f_locals.get('cmd') + if isinstance(cmd, list): + # Fix file case, it may be CL.EXE or cl.exe + argv0 = cmd[0].lower() + if 'cl.exe' in argv0: + lines = [] + # This will not work with "localized" versions + # of MSVC + for line in rec.msg.splitlines(): + if ': warning ' in line: + lines.append(self.parseMessage(line, self.colors.YELLOW)) + elif ': error ' in line: + lines.append(self.parseMessage(line, self.colors.RED)) + elif ': fatal error ' in line: + lines.append(self.parseMessage(line, self.colors.RED + self.colors.BOLD)) + elif ': note: ' in line: + lines.append(self.parseMessage(line, self.colors.CYAN)) + else: + lines.append(line) + rec.msg = "\n".join(lines) + frame = frame.f_back + return Logs.formatter.format(self, rec) + +def options(opt): + Logs.log.handlers[0].setFormatter(ColorMSVCFormatter(Logs.colors)) + diff --git a\/waflib\/extras\/fc_fujitsu.py b\/waflib\/extras\/fc_fujitsu.py new file mode 100644 index 000000000..cae676c20 --- \/dev\/null +++ b\/waflib\/extras\/fc_fujitsu.py @@ -0,0 +1,52 @@ +#! \/usr\/bin\/env python +# encoding: utf-8 +# Detection of the Fujitsu Fortran compiler for ARM64FX + +import re +from waflib.Tools import fc,fc_config,fc_scan +from waflib.Configure import conf +from waflib.Tools.compiler_fc import fc_compiler +fc_compiler['linux'].append('fc_fujitsu') + +@conf +def find_fujitsu(conf): + fc=conf.find_program(['frtpx'],var='FC') + conf.get_fujitsu_version(fc) + conf.env.FC_NAME='FUJITSU' + conf.env.FC_MOD_CAPITALIZATION='lower' + +@conf +def fujitsu_flags(conf): + v=conf.env + v['_FCMODOUTFLAGS']=[] + v['FCFLAGS_DEBUG']=[] + v['FCFLAGS_fcshlib']=[] + v['LINKFLAGS_fcshlib']=[] + v['FCSTLIB_MARKER']='' + v['FCSHLIB_MARKER']='' + +@conf +def get_fujitsu_version(conf,fc): + version_re=re.compile(r"frtpx\s*$FRT$\s*(?P\d+)\.(?P\d+)\.",re.I).search + cmd=fc+['--version'] + out,err=fc_config.getoutput(conf,cmd,stdin=False) + if out: + match=version_re(out) + else: + match=version_re(err) + if not match: + return(False) + conf.fatal('Could not determine the Fujitsu FRT Fortran compiler version.') + else: + k=match.groupdict() + conf.env['FC_VERSION']=(k['major'],k['minor']) + +def configure(conf): + conf.find_fujitsu() + conf.find_program('ar',var='AR') + conf.add_os_flags('ARFLAGS') + if not conf.env.ARFLAGS: + conf.env.ARFLAGS=['rcs'] + conf.fc_flags() + conf.fc_add_flags() + conf.fujitsu_flags() diff --git a\/waflib\/extras\/fc_nfort.py b\/waflib\/extras\/fc_nfort.py new file mode 100644 index 000000000..c25886b8e --- \/dev\/null +++ b\/waflib\/extras\/fc_nfort.py @@ -0,0 +1,52 @@ +#! \/usr\/bin\/env python +# encoding: utf-8 +# Detection of the NEC Fortran compiler for Aurora Tsubasa + +import re +from waflib.Tools import fc,fc_config,fc_scan +from waflib.Configure import conf +from waflib.Tools.compiler_fc import fc_compiler +fc_compiler['linux'].append('fc_nfort') + +@conf +def find_nfort(conf): + fc=conf.find_program(['nfort'],var='FC') + conf.get_nfort_version(fc) + conf.env.FC_NAME='NFORT' + conf.env.FC_MOD_CAPITALIZATION='lower' + +@conf +def nfort_flags(conf): + v=conf.env + v['_FCMODOUTFLAGS']=[] + v['FCFLAGS_DEBUG']=[] + v['FCFLAGS_fcshlib']=[] + v['LINKFLAGS_fcshlib']=[] + v['FCSTLIB_MARKER']='' + v['FCSHLIB_MARKER']='' + +@conf +def get_nfort_version(conf,fc): + version_re=re.compile(r"nfort\s*$NFORT$\s*(?P\d+)\.(?P\d+)\.",re.I).search + cmd=fc+['--version'] + out,err=fc_config.getoutput(conf,cmd,stdin=False) + if out: + match=version_re(out) + else: + match=version_re(err) + if not match: + return(False) + conf.fatal('Could not determine the NEC NFORT Fortran compiler version.') + else: + k=match.groupdict() + conf.env['FC_VERSION']=(k['major'],k['minor']) + +def configure(conf): + conf.find_nfort() + conf.find_program('nar',var='AR') + conf.add_os_flags('ARFLAGS') + if not conf.env.ARFLAGS: + conf.env.ARFLAGS=['rcs'] + conf.fc_flags() + conf.fc_add_flags() + conf.nfort_flags() diff --git a\/waflib\/extras\/genpybind.py b\/waflib\/extras\/genpybind.py new file mode 100644 index 000000000..ac206ee8a --- \/dev\/null +++ b\/waflib\/extras\/genpybind.py @@ -0,0 +1,194 @@ +import os +import pipes +import subprocess +import sys + +from waflib import Logs, Task, Context +from waflib.Tools.c_preproc import scan as scan_impl +# ^-- Note: waflib.extras.gccdeps.scan does not work for us, +# due to its current implementation: +# The -MD flag is injected into the {C,CXX}FLAGS environment variable and +# dependencies are read out in a separate step after compiling by reading +# the .d file saved alongside the object file. +# As the genpybind task refers to a header file that is never compiled itself, +# gccdeps will not be able to extract the list of dependencies. + +from waflib.TaskGen import feature, before_method + + +def join_args(args): + return " ".join(pipes.quote(arg) for arg in args) + + +def configure(cfg): + cfg.load("compiler_cxx") + cfg.load("python") + cfg.check_python_version(minver=(2, 7)) + if not cfg.env.LLVM_CONFIG: + cfg.find_program("llvm-config", var="LLVM_CONFIG") + if not cfg.env.GENPYBIND: + cfg.find_program("genpybind", var="GENPYBIND") + + # find clang reasource dir for builtin headers + cfg.env.GENPYBIND_RESOURCE_DIR = os.path.join( + cfg.cmd_and_log(cfg.env.LLVM_CONFIG + ["--libdir"]).strip(), + "clang", + cfg.cmd_and_log(cfg.env.LLVM_CONFIG + ["--version"]).strip()) + if os.path.exists(cfg.env.GENPYBIND_RESOURCE_DIR): + cfg.msg("Checking clang resource dir", cfg.env.GENPYBIND_RESOURCE_DIR) + else: + cfg.fatal("Clang resource dir not found") + + +@feature("genpybind") +@before_method("process_source") +def generate_genpybind_source(self): + """ + Run genpybind on the headers provided in `source` and compile\/link the + generated code instead. This works by generating the code on the fly and + swapping the source node before `process_source` is run. + """ + # name of module defaults to name of target + module = getattr(self, "module", self.target) + + # create temporary source file in build directory to hold generated code + out = "genpybind-%s.%d.cpp" % (module, self.idx) + out = self.path.get_bld().find_or_declare(out) + + task = self.create_task("genpybind", self.to_nodes(self.source), out) + # used to detect whether CFLAGS or CXXFLAGS should be passed to genpybind + task.features = self.features + task.module = module + # can be used to select definitions to include in the current module + # (when header files are shared by more than one module) + task.genpybind_tags = self.to_list(getattr(self, "genpybind_tags", [])) + # additional include directories + task.includes = self.to_list(getattr(self, "includes", [])) + task.genpybind = self.env.GENPYBIND + + # Tell waf to compile\/link the generated code instead of the headers + # originally passed-in via the `source` parameter. (see `process_source`) + self.source = [out] + + +class genpybind(Task.Task): # pylint: disable=invalid-name + """ + Runs genpybind on headers provided as input to this task. + Generated code will be written to the first (and only) output node. + """ + quiet = True + color = "PINK" + scan = scan_impl + + @staticmethod + def keyword(): + return "Analyzing" + + def run(self): + if not self.inputs: + return + + args = self.find_genpybind() + self._arguments( + resource_dir=self.env.GENPYBIND_RESOURCE_DIR) + + output = self.run_genpybind(args) + + # For debugging \/ log output + pasteable_command = join_args(args) + + # write generated code to file in build directory + # (will be compiled during process_source stage) + (output_node,) = self.outputs + output_node.write("\/\/ {}\n{}\n".format( + pasteable_command.replace("\n", "\n\/\/ "), output)) + + def find_genpybind(self): + return self.genpybind + + def run_genpybind(self, args): + bld = self.generator.bld + + kwargs = dict(cwd=bld.variant_dir) + if hasattr(bld, "log_command"): + bld.log_command(args, kwargs) + else: + Logs.debug("runner: {!r}".format(args)) + proc = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) + stdout, stderr = proc.communicate() + + if not isinstance(stdout, str): + stdout = stdout.decode(sys.stdout.encoding, errors="replace") + if not isinstance(stderr, str): + stderr = stderr.decode(sys.stderr.encoding, errors="replace") + + if proc.returncode != 0: + bld.fatal( + "genpybind returned {code} during the following call:" + "\n{command}\n\n{stdout}\n\n{stderr}".format( + code=proc.returncode, + command=join_args(args), + stdout=stdout, + stderr=stderr, + )) + + if stderr.strip(): + Logs.debug("non-fatal warnings during genpybind run:\n{}".format(stderr)) + + return stdout + + def _include_paths(self): + return self.generator.to_incnodes(self.includes + self.env.INCLUDES) + + def _inputs_as_relative_includes(self): + include_paths = self._include_paths() + relative_includes = [] + for node in self.inputs: + for inc in include_paths: + if node.is_child_of(inc): + relative_includes.append(node.path_from(inc)) + break + else: + self.generator.bld.fatal("could not resolve {}".format(node)) + return relative_includes + + def _arguments(self, genpybind_parse=None, resource_dir=None): + args = [] + relative_includes = self._inputs_as_relative_includes() + is_cxx = "cxx" in self.features + + # options for genpybind + args.extend(["--genpybind-module", self.module]) + if self.genpybind_tags: + args.extend(["--genpybind-tag"] + self.genpybind_tags) + if relative_includes: + args.extend(["--genpybind-include"] + relative_includes) + if genpybind_parse: + args.extend(["--genpybind-parse", genpybind_parse]) + + args.append("--") + + # headers to be processed by genpybind + args.extend(node.abspath() for node in self.inputs) + + args.append("--") + + # options for clang\/genpybind-parse + args.append("-D__GENPYBIND__") + args.append("-xc++" if is_cxx else "-xc") + has_std_argument = False + for flag in self.env["CXXFLAGS" if is_cxx else "CFLAGS"]: + flag = flag.replace("-std=gnu", "-std=c") + if flag.startswith("-std=c"): + has_std_argument = True + args.append(flag) + if not has_std_argument: + args.append("-std=c++14") + args.extend("-I{}".format(n.abspath()) for n in self._include_paths()) + args.extend("-D{}".format(p) for p in self.env.DEFINES) + + # point to clang resource dir, if specified + if resource_dir: + args.append("-resource-dir={}".format(resource_dir)) + + return args diff --git a\/waflib\/extras\/haxe.py b\/waflib\/extras\/haxe.py new file mode 100644 index 000000000..4ff374579 --- \/dev\/null +++ b\/waflib\/extras\/haxe.py @@ -0,0 +1,154 @@ +import re + +from waflib import Utils, Task, Errors, Logs +from waflib.Configure import conf +from waflib.TaskGen import extension, taskgen_method + +HAXE_COMPILERS = { + 'JS': {'tgt': '--js', 'ext_out': ['.js']}, + 'LUA': {'tgt': '--lua', 'ext_out': ['.lua']}, + 'SWF': {'tgt': '--swf', 'ext_out': ['.swf']}, + 'NEKO': {'tgt': '--neko', 'ext_out': ['.n']}, + 'PHP': {'tgt': '--php', 'ext_out': ['.php']}, + 'CPP': {'tgt': '--cpp', 'ext_out': ['.h', '.cpp']}, + 'CPPIA': {'tgt': '--cppia', 'ext_out': ['.cppia']}, + 'CS': {'tgt': '--cs', 'ext_out': ['.cs']}, + 'JAVA': {'tgt': '--java', 'ext_out': ['.java']}, + 'JVM': {'tgt': '--jvm', 'ext_out': ['.jar']}, + 'PYTHON': {'tgt': '--python', 'ext_out': ['.py']}, + 'HL': {'tgt': '--hl', 'ext_out': ['.hl']}, + 'HLC': {'tgt': '--hl', 'ext_out': ['.h', '.c']}, +} + +@conf +def check_haxe_pkg(self, **kw): + self.find_program('haxelib') + libs = kw.get('libs') + if not libs or not (type(libs) == str or (type(libs) == list and all(isinstance(s, str) for s in libs))): + self.fatal('Specify correct libs value in ensure call') + return + fetch = kw.get('fetch') + if not fetch is None and not type(fetch) == bool: + self.fatal('Specify correct fetch value in ensure call') + + libs = [libs] if type(libs) == str else libs + halt = False + for lib in libs: + try: + self.start_msg('Checking for library %s' % lib) + output = self.cmd_and_log(self.env.HAXELIB + ['list', lib]) + except Errors.WafError: + self.end_msg(False) + self.fatal('Can\'t run haxelib list, ensuring halted') + return + + if lib in output: + self.end_msg(lib in output) + else: + if not fetch: + self.end_msg(False) + halt = True + continue + try: + status = self.exec_command(self.env.HAXELIB + ['install', lib]) + if status: + self.end_msg(False) + self.fatal('Can\'t get %s with haxelib, ensuring halted' % lib) + return + else: + self.end_msg('downloaded', color='YELLOW') + except Errors.WafError: + self.end_msg(False) + self.fatal('Can\'t run haxelib install, ensuring halted') + return + postfix = kw.get('uselib_store') or lib.upper() + self.env.append_unique('LIB_' + postfix, lib) + + if halt: + self.fatal('Can\'t find libraries in haxelib list, ensuring halted') + return + +class haxe(Task.Task): + vars = ['HAXE_VERSION', 'HAXE_FLAGS'] + ext_in = ['.hx'] + + def run(self): + cmd = self.env.HAXE + self.env.HAXE_FLAGS_DEFAULT + self.env.HAXE_FLAGS + return self.exec_command(cmd) + +for COMP in HAXE_COMPILERS: + # create runners for each compile target + type("haxe_" + COMP, (haxe,), {'ext_out': HAXE_COMPILERS[COMP]['ext_out']}) + +@taskgen_method +def init_haxe(self): + errmsg = '%s not found, specify correct value' + try: + compiler = HAXE_COMPILERS[self.compiler] + comp_tgt = compiler['tgt'] + comp_mod = '\/main.c' if self.compiler == 'HLC' else '' + except (AttributeError, KeyError): + self.bld.fatal(errmsg % 'COMPILER' + ': ' + ', '.join(HAXE_COMPILERS.keys())) + return + + self.env.append_value( + 'HAXE_FLAGS', + [comp_tgt, self.path.get_bld().make_node(self.target + comp_mod).abspath()]) + if hasattr(self, 'use'): + if not (type(self.use) == str or type(self.use) == list): + self.bld.fatal(errmsg % 'USE') + return + self.use = [self.use] if type(self.use) == str else self.use + + for dep in self.use: + if self.env['LIB_' + dep]: + for lib in self.env['LIB_' + dep]: + self.env.append_value('HAXE_FLAGS', ['-lib', lib]) + + if hasattr(self, 'res'): + if not type(self.res) == str: + self.bld.fatal(errmsg % 'RES') + return + self.env.append_value('HAXE_FLAGS', ['-D', 'resourcesPath=%s' % self.res]) + +@extension('.hx') +def haxe_hook(self, node): + if len(self.source) > 1: + self.bld.fatal('Use separate task generators for multiple files') + return + + src = node + tgt = self.path.get_bld().find_or_declare(self.target) + + self.init_haxe() + self.create_task('haxe_' + self.compiler, src, tgt) + +@conf +def check_haxe(self, mini=None, maxi=None): + self.start_msg('Checking for haxe version') + try: + curr = re.search( + r'(\d+.?)+', + self.cmd_and_log(self.env.HAXE + ['-version'])).group() + except Errors.WafError: + self.end_msg(False) + self.fatal('Can\'t get haxe version') + return + + if mini and Utils.num2ver(curr) < Utils.num2ver(mini): + self.end_msg('wrong', color='RED') + self.fatal('%s is too old, need >= %s' % (curr, mini)) + return + if maxi and Utils.num2ver(curr) > Utils.num2ver(maxi): + self.end_msg('wrong', color='RED') + self.fatal('%s is too new, need <= %s' % (curr, maxi)) + return + self.end_msg(curr, color='GREEN') + self.env.HAXE_VERSION = curr + +def configure(self): + self.env.append_value( + 'HAXE_FLAGS_DEFAULT', + ['-D', 'no-compilation', '-cp', self.path.abspath()]) + Logs.warn('Default flags: %s' % ' '.join(self.env.HAXE_FLAGS_DEFAULT)) + self.find_program('haxe') diff --git a\/waflib\/extras\/msvc_pdb.py b\/waflib\/extras\/msvc_pdb.py new file mode 100644 index 000000000..077656b4f --- \/dev\/null +++ b\/waflib\/extras\/msvc_pdb.py @@ -0,0 +1,46 @@ +#!\/usr\/bin\/env python +# encoding: utf-8 +# Rafaël Kooi 2019 + +from waflib import TaskGen + +@TaskGen.feature('c', 'cxx', 'fc') +@TaskGen.after_method('propagate_uselib_vars') +def add_pdb_per_object(self): + """For msvc\/fortran, specify a unique compile pdb per object, to work + around LNK4099. Flags are updated with a unique \/Fd flag based on the + task output name. This is separate from the link pdb. + """ + if not hasattr(self, 'compiled_tasks'): + return + + link_task = getattr(self, 'link_task', None) + + for task in self.compiled_tasks: + if task.inputs and task.inputs[0].name.lower().endswith('.rc'): + continue + + add_pdb = False + for flagname in ('CFLAGS', 'CXXFLAGS', 'FCFLAGS'): + # several languages may be used at once + for flag in task.env[flagname]: + if flag[1:].lower() == 'zi': + add_pdb = True + break + + if add_pdb: + node = task.outputs[0].change_ext('.pdb') + pdb_flag = '\/Fd:' + node.abspath() + + for flagname in ('CFLAGS', 'CXXFLAGS', 'FCFLAGS'): + buf = [pdb_flag] + for flag in task.env[flagname]: + if flag[1:3] == 'Fd' or flag[1:].lower() == 'fs' or flag[1:].lower() == 'mp': + continue + buf.append(flag) + task.env[flagname] = buf + + if link_task and not node in link_task.dep_nodes: + link_task.dep_nodes.append(node) + if not node in task.outputs: + task.outputs.append(node) diff --git a\/waflib\/extras\/sphinx.py b\/waflib\/extras\/sphinx.py new file mode 100644 index 000000000..08f3cfd8a --- \/dev\/null +++ b\/waflib\/extras\/sphinx.py @@ -0,0 +1,120 @@ +"""Support for Sphinx documentation + +This is a wrapper for sphinx-build program. Please note that sphinx-build supports only +one output format at a time, but the tool can create multiple tasks to handle more. +The output formats can be passed via the sphinx_output_format, which is an array of +strings. For backwards compatibility if only one output is needed, it can be passed +as a single string. +The default output format is html. + +Specific formats can be installed in different directories by specifying the +install_path_ attribute. If not defined, the standard install_path +will be used instead. + +Example wscript: + +def configure(cnf): + conf.load('sphinx') + +def build(bld): + bld( + features='sphinx', + sphinx_source='sources', # path to source directory + sphinx_options='-a -v', # sphinx-build program additional options + sphinx_output_format=['html', 'man'], # output format of sphinx documentation + install_path_man='${DOCDIR}\/man' # put man pages in a specific directory + ) + +""" + +from waflib.Node import Node +from waflib import Utils +from waflib import Task +from waflib.TaskGen import feature, after_method + + +def configure(cnf): + """Check if sphinx-build program is available and loads gnu_dirs tool.""" + cnf.find_program('sphinx-build', var='SPHINX_BUILD', mandatory=False) + cnf.load('gnu_dirs') + + +@feature('sphinx') +def build_sphinx(self): + """Builds sphinx sources. + """ + if not self.env.SPHINX_BUILD: + self.bld.fatal('Program SPHINX_BUILD not defined.') + if not getattr(self, 'sphinx_source', None): + self.bld.fatal('Attribute sphinx_source not defined.') + if not isinstance(self.sphinx_source, Node): + self.sphinx_source = self.path.find_node(self.sphinx_source) + if not self.sphinx_source: + self.bld.fatal('Can\'t find sphinx_source: %r' % self.sphinx_source) + + # In the taskgen we have the complete list of formats + Utils.def_attrs(self, sphinx_output_format='html') + self.sphinx_output_format = Utils.to_list(self.sphinx_output_format) + + self.env.SPHINX_OPTIONS = getattr(self, 'sphinx_options', []) + + for source_file in self.sphinx_source.ant_glob('**\/*'): + self.bld.add_manual_dependency(self.sphinx_source, source_file) + + for cfmt in self.sphinx_output_format: + sphinx_build_task = self.create_task('SphinxBuildingTask') + sphinx_build_task.set_inputs(self.sphinx_source) + # In task we keep the specific format this task is generating + sphinx_build_task.env.SPHINX_OUTPUT_FORMAT = cfmt + + # the sphinx-build results are in directory + sphinx_build_task.sphinx_output_directory = self.path.get_bld().make_node(cfmt) + sphinx_build_task.set_outputs(sphinx_build_task.sphinx_output_directory) + sphinx_build_task.sphinx_output_directory.mkdir() + + Utils.def_attrs(sphinx_build_task, install_path=getattr(self, 'install_path_' + cfmt, getattr(self, 'install_path', get_install_path(sphinx_build_task)))) + + +def get_install_path(object): + if object.env.SPHINX_OUTPUT_FORMAT == 'man': + return object.env.MANDIR + elif object.env.SPHINX_OUTPUT_FORMAT == 'info': + return object.env.INFODIR + else: + return object.env.DOCDIR + + +class SphinxBuildingTask(Task.Task): + color = 'BOLD' + run_str = '${SPHINX_BUILD} -M ${SPHINX_OUTPUT_FORMAT} ${SRC} ${TGT} -d ${TGT[0].bld_dir()}\/doctrees-${SPHINX_OUTPUT_FORMAT} ${SPHINX_OPTIONS}' + + def keyword(self): + return 'Compiling (%s)' % self.env.SPHINX_OUTPUT_FORMAT + + def runnable_status(self): + + for x in self.run_after: + if not x.hasrun: + return Task.ASK_LATER + + self.signature() + ret = Task.Task.runnable_status(self) + if ret == Task.SKIP_ME: + # in case the files were removed + self.add_install() + return ret + + + def post_run(self): + self.add_install() + return Task.Task.post_run(self) + + + def add_install(self): + nodes = self.sphinx_output_directory.ant_glob('**\/*', quiet=True) + self.outputs += nodes + self.generator.add_install_files(install_to=self.install_path, + install_from=nodes, + postpone=False, + cwd=self.sphinx_output_directory.make_node(self.env.SPHINX_OUTPUT_FORMAT), + relative_trick=True) diff --git a\/waflib\/extras\/wafcache.py b\/waflib\/extras\/wafcache.py new file mode 100644 index 000000000..30ac3ef51 --- \/dev\/null +++ b\/waflib\/extras\/wafcache.py @@ -0,0 +1,648 @@ +#! \/usr\/bin\/env python +# encoding: utf-8 +# Thomas Nagy, 2019 (ita) + +""" +Filesystem-based cache system to share and re-use build artifacts + +Cache access operations (copy to and from) are delegated to +independent pre-forked worker subprocesses. + +The following environment variables may be set: +* WAFCACHE: several possibilities: + - File cache: + absolute path of the waf cache (~\/.cache\/wafcache_user, + where `user` represents the currently logged-in user) + - URL to a cache server, for example: + export WAFCACHE=http:\/\/localhost:8080\/files\/ + in that case, GET\/POST requests are made to urls of the form + http:\/\/localhost:8080\/files\/000000000\/0 (cache management is delegated to the server) + - GCS, S3 or MINIO bucket + gs:\/\/my-bucket\/ (uses gsutil command line tool or WAFCACHE_CMD) + s3:\/\/my-bucket\/ (uses aws command line tool or WAFCACHE_CMD) + minio:\/\/my-bucket\/ (uses mc command line tool or WAFCACHE_CMD) +* WAFCACHE_CMD: bucket upload\/download command, for example: + WAFCACHE_CMD="gsutil cp %{SRC} %{TGT}" + Note that the WAFCACHE bucket value is used for the source or destination + depending on the operation (upload or download). For example, with: + WAFCACHE="gs:\/\/mybucket\/" + the following commands may be run: + gsutil cp build\/myprogram gs:\/\/mybucket\/aa\/aaaaa\/1 + gsutil cp gs:\/\/mybucket\/bb\/bbbbb\/2 build\/somefile +* WAFCACHE_NO_PUSH: if set, disables pushing to the cache +* WAFCACHE_VERBOSITY: if set, displays more detailed cache operations +* WAFCACHE_STATS: if set, displays cache usage statistics on exit + +File cache specific options: + Files are copied using hard links by default; if the cache is located + onto another partition, the system switches to file copies instead. +* WAFCACHE_TRIM_MAX_FOLDER: maximum amount of tasks to cache (1M) +* WAFCACHE_EVICT_MAX_BYTES: maximum amount of cache size in bytes (10GB) +* WAFCACHE_EVICT_INTERVAL_MINUTES: minimum time interval to try + and trim the cache (3 minutes) + +Upload specific options: +* WAFCACHE_ASYNC_WORKERS: define a number of workers to upload results asynchronously + this may improve build performance with many\/long file uploads + the default is unset (synchronous uploads) +* WAFCACHE_ASYNC_NOWAIT: do not wait for uploads to complete (default: False) + this requires asynchonous uploads to have an effect + +Usage:: + + def build(bld): + bld.load('wafcache') + ... + +To troubleshoot:: + + waf clean build --zone=wafcache +""" + +import atexit, base64, errno, fcntl, getpass, os, re, shutil, sys, time, threading, traceback, urllib3, shlex +try: + import subprocess32 as subprocess +except ImportError: + import subprocess + +base_cache = os.path.expanduser('~\/.cache\/') +if not os.path.isdir(base_cache): + base_cache = '\/tmp\/' +default_wafcache_dir = os.path.join(base_cache, 'wafcache_' + getpass.getuser()) + +CACHE_DIR = os.environ.get('WAFCACHE', default_wafcache_dir) +WAFCACHE_CMD = os.environ.get('WAFCACHE_CMD') +TRIM_MAX_FOLDERS = int(os.environ.get('WAFCACHE_TRIM_MAX_FOLDER', 1000000)) +EVICT_INTERVAL_MINUTES = int(os.environ.get('WAFCACHE_EVICT_INTERVAL_MINUTES', 3)) +EVICT_MAX_BYTES = int(os.environ.get('WAFCACHE_EVICT_MAX_BYTES', 10**10)) +WAFCACHE_NO_PUSH = 1 if os.environ.get('WAFCACHE_NO_PUSH') else 0 +WAFCACHE_VERBOSITY = 1 if os.environ.get('WAFCACHE_VERBOSITY') else 0 +WAFCACHE_STATS = 1 if os.environ.get('WAFCACHE_STATS') else 0 +WAFCACHE_ASYNC_WORKERS = os.environ.get('WAFCACHE_ASYNC_WORKERS') +WAFCACHE_ASYNC_NOWAIT = os.environ.get('WAFCACHE_ASYNC_NOWAIT') +OK = "ok" + +re_waf_cmd = re.compile('(?P%{SRC})|(?P%{TGT})') + +try: + import cPickle +except ImportError: + import pickle as cPickle + +if __name__ != '__main__': + from waflib import Task, Logs, Utils, Build + +def can_retrieve_cache(self): + """ + New method for waf Task classes + """ + if not self.outputs: + return False + + self.cached = False + + sig = self.signature() + ssig = Utils.to_hex(self.uid() + sig) + + if WAFCACHE_STATS: + self.generator.bld.cache_reqs += 1 + + files_to = [node.abspath() for node in self.outputs] + proc = get_process() + err = cache_command(proc, ssig, [], files_to) + process_pool.append(proc) + if err.startswith(OK): + if WAFCACHE_VERBOSITY: + Logs.pprint('CYAN', ' Fetched %r from cache' % files_to) + else: + Logs.debug('wafcache: fetched %r from cache', files_to) + if WAFCACHE_STATS: + self.generator.bld.cache_hits += 1 + else: + if WAFCACHE_VERBOSITY: + Logs.pprint('YELLOW', ' No cache entry %s' % files_to) + else: + Logs.debug('wafcache: No cache entry %s: %s', files_to, err) + return False + + self.cached = True + return True + +def put_files_cache(self): + """ + New method for waf Task classes + """ + if WAFCACHE_NO_PUSH or getattr(self, 'cached', None) or not self.outputs: + return + + files_from = [] + for node in self.outputs: + path = node.abspath() + if not os.path.isfile(path): + return + files_from.append(path) + + bld = self.generator.bld + old_sig = self.signature() + + for node in self.inputs: + try: + del node.ctx.cache_sig[node] + except KeyError: + pass + + delattr(self, 'cache_sig') + sig = self.signature() + + def _async_put_files_cache(bld, ssig, files_from): + proc = get_process() + if WAFCACHE_ASYNC_WORKERS: + with bld.wafcache_lock: + if bld.wafcache_stop: + process_pool.append(proc) + return + bld.wafcache_procs.add(proc) + + err = cache_command(proc, ssig, files_from, []) + process_pool.append(proc) + if err.startswith(OK): + if WAFCACHE_VERBOSITY: + Logs.pprint('CYAN', ' Successfully uploaded %s to cache' % files_from) + else: + Logs.debug('wafcache: Successfully uploaded %r to cache', files_from) + if WAFCACHE_STATS: + bld.cache_puts += 1 + else: + if WAFCACHE_VERBOSITY: + Logs.pprint('RED', ' Error caching step results %s: %s' % (files_from, err)) + else: + Logs.debug('wafcache: Error caching results %s: %s', files_from, err) + + if old_sig == sig: + ssig = Utils.to_hex(self.uid() + sig) + if WAFCACHE_ASYNC_WORKERS: + fut = bld.wafcache_executor.submit(_async_put_files_cache, bld, ssig, files_from) + bld.wafcache_uploads.append(fut) + else: + _async_put_files_cache(bld, ssig, files_from) + else: + Logs.debug('wafcache: skipped %r upload due to late input modifications %r', self.outputs, self.inputs) + + bld.task_sigs[self.uid()] = self.cache_sig + +def hash_env_vars(self, env, vars_lst): + """ + Reimplement BuildContext.hash_env_vars so that the resulting hash does not depend on local paths + """ + if not env.table: + env = env.parent + if not env: + return Utils.SIG_NIL + + idx = str(id(env)) + str(vars_lst) + try: + cache = self.cache_env + except AttributeError: + cache = self.cache_env = {} + else: + try: + return self.cache_env[idx] + except KeyError: + pass + + v = str([env[a] for a in vars_lst]) + v = v.replace(self.srcnode.abspath().__repr__()[:-1], '') + m = Utils.md5() + m.update(v.encode()) + ret = m.digest() + + Logs.debug('envhash: %r %r', ret, v) + + cache[idx] = ret + + return ret + +def uid(self): + """ + Reimplement Task.uid() so that the signature does not depend on local paths + """ + try: + return self.uid_ + except AttributeError: + m = Utils.md5() + src = self.generator.bld.srcnode + up = m.update + up(self.__class__.__name__.encode()) + for x in self.inputs + self.outputs: + up(x.path_from(src).encode()) + self.uid_ = m.digest() + return self.uid_ + + +def make_cached(cls): + """ + Enable the waf cache for a given task class + """ + if getattr(cls, 'nocache', None) or getattr(cls, 'has_cache', False): + return + + full_name = "%s.%s" % (cls.__module__, cls.__name__) + if full_name in ('waflib.Tools.ccroot.vnum', 'waflib.Build.inst'): + return + + m1 = getattr(cls, 'run', None) + def run(self): + if getattr(self, 'nocache', False): + return m1(self) + if self.can_retrieve_cache(): + return 0 + return m1(self) + cls.run = run + + m2 = getattr(cls, 'post_run', None) + def post_run(self): + if getattr(self, 'nocache', False): + return m2(self) + ret = m2(self) + self.put_files_cache() + return ret + cls.post_run = post_run + cls.has_cache = True + +process_pool = [] +def get_process(): + """ + Returns a worker process that can process waf cache commands + The worker process is assumed to be returned to the process pool when unused + """ + try: + return process_pool.pop() + except IndexError: + filepath = os.path.dirname(os.path.abspath(__file__)) + os.sep + 'wafcache.py' + cmd = [sys.executable, '-c', Utils.readf(filepath)] + return subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, bufsize=0) + +def atexit_pool(): + for proc in process_pool: + proc.kill() +atexit.register(atexit_pool) + +def build(bld): + """ + Called during the build process to enable file caching + """ + + if WAFCACHE_ASYNC_WORKERS: + try: + num_workers = int(WAFCACHE_ASYNC_WORKERS) + except ValueError: + Logs.warn('Invalid WAFCACHE_ASYNC_WORKERS specified: %r' % WAFCACHE_ASYNC_WORKERS) + else: + from concurrent.futures import ThreadPoolExecutor + bld.wafcache_executor = ThreadPoolExecutor(max_workers=num_workers) + bld.wafcache_uploads = [] + bld.wafcache_procs = set([]) + bld.wafcache_stop = False + bld.wafcache_lock = threading.Lock() + + def finalize_upload_async(bld): + if WAFCACHE_ASYNC_NOWAIT: + with bld.wafcache_lock: + bld.wafcache_stop = True + + for fut in reversed(bld.wafcache_uploads): + fut.cancel() + + for proc in bld.wafcache_procs: + proc.kill() + + bld.wafcache_procs.clear() + else: + Logs.pprint('CYAN', '... waiting for wafcache uploads to complete (%s uploads)' % len(bld.wafcache_uploads)) + bld.wafcache_executor.shutdown(wait=True) + bld.add_post_fun(finalize_upload_async) + + if WAFCACHE_STATS: + # Init counter for statistics and hook to print results at the end + bld.cache_reqs = bld.cache_hits = bld.cache_puts = 0 + + def printstats(bld): + hit_ratio = 0 + if bld.cache_reqs > 0: + hit_ratio = (bld.cache_hits \/ bld.cache_reqs) * 100 + Logs.pprint('CYAN', ' wafcache stats: %s requests, %s hits (ratio: %.2f%%), %s writes' % + (bld.cache_reqs, bld.cache_hits, hit_ratio, bld.cache_puts) ) + bld.add_post_fun(printstats) + + if process_pool: + # already called once + return + + # pre-allocation + processes = [get_process() for x in range(bld.jobs)] + process_pool.extend(processes) + + Task.Task.can_retrieve_cache = can_retrieve_cache + Task.Task.put_files_cache = put_files_cache + Task.Task.uid = uid + Build.BuildContext.hash_env_vars = hash_env_vars + for x in reversed(list(Task.classes.values())): + make_cached(x) + +def cache_command(proc, sig, files_from, files_to): + """ + Create a command for cache worker processes, returns a pickled + base64-encoded tuple containing the task signature, a list of files to + cache and a list of files files to get from cache (one of the lists + is assumed to be empty) + """ + obj = base64.b64encode(cPickle.dumps([sig, files_from, files_to])) + proc.stdin.write(obj) + proc.stdin.write('\n'.encode()) + proc.stdin.flush() + obj = proc.stdout.readline() + if not obj: + raise OSError('Preforked sub-process %r died' % proc.pid) + return cPickle.loads(base64.b64decode(obj)) + +try: + copyfun = os.link +except NameError: + copyfun = shutil.copy2 + +def atomic_copy(orig, dest): + """ + Copy files to the cache, the operation is atomic for a given file + """ + global copyfun + tmp = dest + '.tmp' + up = os.path.dirname(dest) + try: + os.makedirs(up) + except OSError: + pass + + try: + copyfun(orig, tmp) + except OSError as e: + if e.errno == errno.EXDEV: + copyfun = shutil.copy2 + copyfun(orig, tmp) + else: + raise + os.rename(tmp, dest) + +def lru_trim(): + """ + the cache folders take the form: + `CACHE_DIR\/0b\/0b180f82246d726ece37c8ccd0fb1cde2650d7bfcf122ec1f169079a3bfc0ab9` + they are listed in order of last access, and then removed + until the amount of folders is within TRIM_MAX_FOLDERS and the total space + taken by files is less than EVICT_MAX_BYTES + """ + lst = [] + for up in os.listdir(CACHE_DIR): + if len(up) == 2: + sub = os.path.join(CACHE_DIR, up) + for hval in os.listdir(sub): + path = os.path.join(sub, hval) + + size = 0 + for fname in os.listdir(path): + try: + size += os.lstat(os.path.join(path, fname)).st_size + except OSError: + pass + lst.append((os.stat(path).st_mtime, size, path)) + + lst.sort(key=lambda x: x[0]) + lst.reverse() + + tot = sum(x[1] for x in lst) + while tot > EVICT_MAX_BYTES or len(lst) > TRIM_MAX_FOLDERS: + _, tmp_size, path = lst.pop() + tot -= tmp_size + + tmp = path + '.remove' + try: + shutil.rmtree(tmp) + except OSError: + pass + try: + os.rename(path, tmp) + except OSError: + sys.stderr.write('Could not rename %r to %r\n' % (path, tmp)) + else: + try: + shutil.rmtree(tmp) + except OSError: + sys.stderr.write('Could not remove %r\n' % tmp) + sys.stderr.write("Cache trimmed: %r bytes in %r folders left\n" % (tot, len(lst))) + + +def lru_evict(): + """ + Reduce the cache size + """ + lockfile = os.path.join(CACHE_DIR, 'all.lock') + try: + st = os.stat(lockfile) + except EnvironmentError as e: + if e.errno == errno.ENOENT: + with open(lockfile, 'w') as f: + f.write('') + return + else: + raise + + if st.st_mtime < time.time() - EVICT_INTERVAL_MINUTES * 60: + # check every EVICT_INTERVAL_MINUTES minutes if the cache is too big + # OCLOEXEC is unnecessary because no processes are spawned + fd = os.open(lockfile, os.O_RDWR | os.O_CREAT, 0o755) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except EnvironmentError: + if WAFCACHE_VERBOSITY: + sys.stderr.write('wafcache: another cleaning process is running\n') + else: + # now dow the actual cleanup + lru_trim() + os.utime(lockfile, None) + finally: + os.close(fd) + +class netcache(object): + def __init__(self): + self.http = urllib3.PoolManager() + + def url_of(self, sig, i): + return "%s\/%s\/%s" % (CACHE_DIR, sig, i) + + def upload(self, file_path, sig, i): + url = self.url_of(sig, i) + with open(file_path, 'rb') as f: + file_data = f.read() + r = self.http.request('POST', url, timeout=60, + fields={ 'file': ('%s\/%s' % (sig, i), file_data), }) + if r.status >= 400: + raise OSError("Invalid status %r %r" % (url, r.status)) + + def download(self, file_path, sig, i): + url = self.url_of(sig, i) + with self.http.request('GET', url, preload_content=False, timeout=60) as inf: + if inf.status >= 400: + raise OSError("Invalid status %r %r" % (url, inf.status)) + with open(file_path, 'wb') as out: + shutil.copyfileobj(inf, out) + + def copy_to_cache(self, sig, files_from, files_to): + try: + for i, x in enumerate(files_from): + if not os.path.islink(x): + self.upload(x, sig, i) + except Exception: + return traceback.format_exc() + return OK + + def copy_from_cache(self, sig, files_from, files_to): + try: + for i, x in enumerate(files_to): + self.download(x, sig, i) + except Exception: + return traceback.format_exc() + return OK + +class fcache(object): + def __init__(self): + if not os.path.exists(CACHE_DIR): + try: + os.makedirs(CACHE_DIR) + except OSError: + pass + if not os.path.exists(CACHE_DIR): + raise ValueError('Could not initialize the cache directory') + + def copy_to_cache(self, sig, files_from, files_to): + """ + Copy files to the cache, existing files are overwritten, + and the copy is atomic only for a given file, not for all files + that belong to a given task object + """ + try: + for i, x in enumerate(files_from): + dest = os.path.join(CACHE_DIR, sig[:2], sig, str(i)) + atomic_copy(x, dest) + except Exception: + return traceback.format_exc() + else: + # attempt trimming if caching was successful: + # we may have things to trim! + try: + lru_evict() + except Exception: + return traceback.format_exc() + return OK + + def copy_from_cache(self, sig, files_from, files_to): + """ + Copy files from the cache + """ + try: + for i, x in enumerate(files_to): + orig = os.path.join(CACHE_DIR, sig[:2], sig, str(i)) + atomic_copy(orig, x) + + # success! update the cache time + os.utime(os.path.join(CACHE_DIR, sig[:2], sig), None) + except Exception: + return traceback.format_exc() + return OK + +class bucket_cache(object): + def bucket_copy(self, source, target): + if WAFCACHE_CMD: + def replacer(match): + if match.group('src'): + return source + elif match.group('tgt'): + return target + cmd = [re_waf_cmd.sub(replacer, x) for x in shlex.split(WAFCACHE_CMD)] + elif CACHE_DIR.startswith('s3:\/\/'): + cmd = ['aws', 's3', 'cp', source, target] + elif CACHE_DIR.startswith('gs:\/\/'): + cmd = ['gsutil', 'cp', source, target] + else: + cmd = ['mc', 'cp', source, target] + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = proc.communicate() + if proc.returncode: + raise OSError('Error copy %r to %r using: %r (exit %r):\n out:%s\n err:%s' % ( + source, target, cmd, proc.returncode, out.decode(errors='replace'), err.decode(errors='replace'))) + + def copy_to_cache(self, sig, files_from, files_to): + try: + for i, x in enumerate(files_from): + dest = os.path.join(CACHE_DIR, sig[:2], sig, str(i)) + self.bucket_copy(x, dest) + except Exception: + return traceback.format_exc() + return OK + + def copy_from_cache(self, sig, files_from, files_to): + try: + for i, x in enumerate(files_to): + orig = os.path.join(CACHE_DIR, sig[:2], sig, str(i)) + self.bucket_copy(orig, x) + except EnvironmentError: + return traceback.format_exc() + return OK + +def loop(service): + """ + This function is run when this file is run as a standalone python script, + it assumes a parent process that will communicate the commands to it + as pickled-encoded tuples (one line per command) + + The commands are to copy files to the cache or copy files from the + cache to a target destination + """ + # one operation is performed at a single time by a single process + # therefore stdin never has more than one line + txt = sys.stdin.readline().strip() + if not txt: + # parent process probably ended + sys.exit(1) + ret = OK + + [sig, files_from, files_to] = cPickle.loads(base64.b64decode(txt)) + if files_from: + # TODO return early when pushing files upstream + ret = service.copy_to_cache(sig, files_from, files_to) + elif files_to: + # the build process waits for workers to (possibly) obtain files from the cache + ret = service.copy_from_cache(sig, files_from, files_to) + else: + ret = "Invalid command" + + obj = base64.b64encode(cPickle.dumps(ret)) + sys.stdout.write(obj.decode()) + sys.stdout.write('\n') + sys.stdout.flush() + +if __name__ == '__main__': + if CACHE_DIR.startswith('s3:\/\/') or CACHE_DIR.startswith('gs:\/\/') or CACHE_DIR.startswith('minio:\/\/'): + if CACHE_DIR.startswith('minio:\/\/'): + CACHE_DIR = CACHE_DIR[8:] # minio doesn't need the protocol part, uses config aliases + service = bucket_cache() + elif CACHE_DIR.startswith('http'): + service = netcache() + else: + service = fcache() + while 1: + try: + loop(service) + except KeyboardInterrupt: + break + diff --git a\/waflib\/extras\/xcode6.py b\/waflib\/extras\/xcode6.py index 91bbff181..c5b309120 100644 --- a\/waflib\/extras\/xcode6.py +++ b\/waflib\/extras\/xcode6.py @@ -99,7 +99,7 @@ def delete_invalid_values(dct): ... } 'Release': { - 'ARCHS' x86_64' + 'ARCHS': x86_64' ... } } @@ -163,12 +163,12 @@ def tostring(self, value): result = result + "\t\t}" return result elif isinstance(value, str): - return "\"%s\"" % value + return '"%s"' % value.replace('"', '\\\\\\"') elif isinstance(value, list): result = "(\n" for i in value: - result = result + "\t\t\t%s,\n" % self.tostring(i) - result = result + "\t\t)" + result = result + "\t\t\t\t%s,\n" % self.tostring(i) + result = result + "\t\t\t)" return result elif isinstance(value, XCodeNode): return value._id @@ -565,13 +565,13 @@ def process_xcode(self): # Override target specific build settings bldsettings = { 'HEADER_SEARCH_PATHS': ['$(inherited)'] + self.env['INCPATHS'], - 'LIBRARY_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.LIBPATH) + Utils.to_list(self.env.STLIBPATH) + Utils.to_list(self.env.LIBDIR) , + 'LIBRARY_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.LIBPATH) + Utils.to_list(self.env.STLIBPATH) + Utils.to_list(self.env.LIBDIR), 'FRAMEWORK_SEARCH_PATHS': ['$(inherited)'] + Utils.to_list(self.env.FRAMEWORKPATH), - 'OTHER_LDFLAGS': libs + ' ' + frameworks, - 'OTHER_LIBTOOLFLAGS': bld.env['LINKFLAGS'], + 'OTHER_LDFLAGS': libs + ' ' + frameworks + ' ' + ' '.join(bld.env['LINKFLAGS']), 'OTHER_CPLUSPLUSFLAGS': Utils.to_list(self.env['CXXFLAGS']), 'OTHER_CFLAGS': Utils.to_list(self.env['CFLAGS']), - 'INSTALL_PATH': [] + 'INSTALL_PATH': [], + 'GCC_PREPROCESSOR_DEFINITIONS': self.env['DEFINES'] } # Install path @@ -591,7 +591,7 @@ def process_xcode(self): # The keys represents different build configuration, e.g. Debug, Release and so on.. # Insert our generated build settings to all configuration names - keys = set(settings.keys() + bld.env.PROJ_CONFIGURATION.keys()) + keys = set(settings.keys()) | set(bld.env.PROJ_CONFIGURATION.keys()) for k in keys: if k in settings: settings[k].update(bldsettings) diff --git a\/waflib\/fixpy2.py b\/waflib\/fixpy2.py index 24176e066..c99bff4b9 100644 --- a\/waflib\/fixpy2.py +++ b\/waflib\/fixpy2.py @@ -56,7 +56,7 @@ def r1(code): @subst('Runner.py') def r4(code): "generator syntax" - return code.replace('next(self.biter)', 'self.biter.next()') + return code.replace('next(self.biter)', 'self.biter.next()').replace('self.daemon = True', 'self.setDaemon(1)') @subst('Context.py') def r5(code): diff --git a\/waflib\/processor.py b\/waflib\/processor.py index 2eecf3bd9..eff2e69ad 100755 --- a\/waflib\/processor.py +++ b\/waflib\/processor.py @@ -27,6 +27,10 @@ def run(): [cmd, kwargs, cargs] = cPickle.loads(base64.b64decode(txt)) cargs = cargs or {} + if not 'close_fds' in kwargs: + # workers have no fds + kwargs['close_fds'] = False + ret = 1 out, err, ex, trace = (None, None, None, None) try:

Other Versions#

Patches#