Merge branch 'dev'5.3.0master

1 files changed, 279 insertions, 101 deletions

diff --git a/scripts/gen_travis.py b/scripts/gen_travis.py
index f1478c62..4366a066 100755
--- a/scripts/gen_travis.py
+++ b/scripts/gen_travis.py
@@ -1,149 +1,327 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
-from itertools import combinations
+from itertools import combinations, chain
+from enum import Enum, auto
 
-travis_template = """\
-language: generic
-dist: precise
 
-matrix:
+LINUX = 'linux'
+OSX = 'osx'
+WINDOWS = 'windows'
+FREEBSD = 'freebsd'
+
+
+AMD64 = 'amd64'
+ARM64 = 'arm64'
+PPC64LE = 'ppc64le'
+
+
+TRAVIS_TEMPLATE = """\
+# This config file is generated by ./scripts/gen_travis.py.
+# Do not edit by hand.
+
+# We use 'minimal', because 'generic' makes Windows VMs hang at startup. Also
+# the software provided by 'generic' is simply not needed for our tests.
+# Differences are explained here:
+# https://docs.travis-ci.com/user/languages/minimal-and-generic/
+language: minimal
+dist: focal
+
+jobs:
   include:
-%s
+{jobs}
+
+before_install:
+  - |-
+    if test -f "./scripts/$TRAVIS_OS_NAME/before_install.sh"; then
+      source ./scripts/$TRAVIS_OS_NAME/before_install.sh
+    fi
 
 before_script:
-  - autoconf
-  - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script
-  - ./configure ${COMPILER_FLAGS:+ \
-      CC="$CC $COMPILER_FLAGS" \
-      CXX="$CXX $COMPILER_FLAGS" } \
-      $CONFIGURE_FLAGS
-  - make -j3
-  - make -j3 tests
+  - |-
+    if test -f "./scripts/$TRAVIS_OS_NAME/before_script.sh"; then
+      source ./scripts/$TRAVIS_OS_NAME/before_script.sh
+    else
+      scripts/gen_travis.py > travis_script && diff .travis.yml travis_script
+      autoconf
+      # If COMPILER_FLAGS are not empty, add them to CC and CXX
+      ./configure ${{COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" \
+CXX="$CXX $COMPILER_FLAGS"}} $CONFIGURE_FLAGS
+      make -j3
+      make -j3 tests
+    fi
 
 script:
-  - make check
+  - |-
+    if test -f "./scripts/$TRAVIS_OS_NAME/script.sh"; then
+      source ./scripts/$TRAVIS_OS_NAME/script.sh
+    else
+      make check
+    fi
 """
 
+
+class Option(object):
+    class Type:
+        COMPILER = auto()
+        COMPILER_FLAG = auto()
+        CONFIGURE_FLAG = auto()
+        MALLOC_CONF = auto()
+        FEATURE = auto()
+
+    def __init__(self, type, value):
+        self.type = type
+        self.value = value
+
+    @staticmethod
+    def as_compiler(value):
+        return Option(Option.Type.COMPILER, value)
+
+    @staticmethod
+    def as_compiler_flag(value):
+        return Option(Option.Type.COMPILER_FLAG, value)
+
+    @staticmethod
+    def as_configure_flag(value):
+        return Option(Option.Type.CONFIGURE_FLAG, value)
+
+    @staticmethod
+    def as_malloc_conf(value):
+        return Option(Option.Type.MALLOC_CONF, value)
+
+    @staticmethod
+    def as_feature(value):
+        return Option(Option.Type.FEATURE, value)
+
+    def __eq__(self, obj):
+        return (isinstance(obj, Option) and obj.type == self.type
+                and obj.value == self.value)
+
+
 # The 'default' configuration is gcc, on linux, with no compiler or configure
 # flags.  We also test with clang, -m32, --enable-debug, --enable-prof,
 # --disable-stats, and --with-malloc-conf=tcache:false.  To avoid abusing
 # travis though, we don't test all 2**7 = 128 possible combinations of these;
 # instead, we only test combinations of up to 2 'unusual' settings, under the
 # hope that bugs involving interactions of such settings are rare.
-# Things at once, for C(7, 0) + C(7, 1) + C(7, 2) = 29
 MAX_UNUSUAL_OPTIONS = 2
 
-os_default = 'linux'
-os_unusual = 'osx'
 
-compilers_default = 'CC=gcc CXX=g++'
-compilers_unusual = 'CC=clang CXX=clang++'
+GCC = Option.as_compiler('CC=gcc CXX=g++')
+CLANG = Option.as_compiler('CC=clang CXX=clang++')
+CL = Option.as_compiler('CC=cl.exe CXX=cl.exe')
+
 
-compiler_flag_unusuals = ['-m32']
+compilers_unusual = [CLANG,]
 
-configure_flag_unusuals = [
+
+CROSS_COMPILE_32BIT = Option.as_feature('CROSS_COMPILE_32BIT')
+feature_unusuals = [CROSS_COMPILE_32BIT]
+
+
+configure_flag_unusuals = [Option.as_configure_flag(opt) for opt in (
     '--enable-debug',
     '--enable-prof',
     '--disable-stats',
     '--disable-libdl',
     '--enable-opt-safety-checks',
-]
+    '--with-lg-page=16',
+)]
 
-malloc_conf_unusuals = [
+
+malloc_conf_unusuals = [Option.as_malloc_conf(opt) for opt in (
     'tcache:false',
     'dss:primary',
     'percpu_arena:percpu',
     'background_thread:true',
-]
+)]
 
-all_unusuals = (
-    [os_unusual] + [compilers_unusual] + compiler_flag_unusuals
-    + configure_flag_unusuals + malloc_conf_unusuals
-)
 
-unusual_combinations_to_test = []
-for i in xrange(MAX_UNUSUAL_OPTIONS + 1):
-    unusual_combinations_to_test += combinations(all_unusuals, i)
+all_unusuals = (compilers_unusual + feature_unusuals
+    + configure_flag_unusuals + malloc_conf_unusuals)
 
-gcc_multilib_set = False
-# Formats a job from a combination of flags
-def format_job(combination):
-    global gcc_multilib_set
 
-    os = os_unusual if os_unusual in combination else os_default
-    compilers = compilers_unusual if compilers_unusual in combination else compilers_default
+def get_extra_cflags(os, compiler):
+    if os == FREEBSD:
+        return []
+
+    if os == WINDOWS:
+        # For non-CL compilers under Windows (for now it's only MinGW-GCC),
+        # -fcommon needs to be specified to correctly handle multiple
+        # 'malloc_conf' symbols and such, which are declared weak under Linux.
+        # Weak symbols don't work with MinGW-GCC.
+        if compiler != CL.value:
+            return ['-fcommon']
+        else:
+            return []
+
+    # We get some spurious errors when -Warray-bounds is enabled.
+    extra_cflags = ['-Werror', '-Wno-array-bounds']
+    if compiler == CLANG.value or os == OSX:
+        extra_cflags += [
+            '-Wno-unknown-warning-option',
+            '-Wno-ignored-attributes'
+        ]
+    if os == OSX:
+        extra_cflags += [
+            '-Wno-deprecated-declarations',
+        ]
+    return extra_cflags
 
-    compiler_flags = [x for x in combination if x in compiler_flag_unusuals]
-    configure_flags = [x for x in combination if x in configure_flag_unusuals]
-    malloc_conf = [x for x in combination if x in malloc_conf_unusuals]
 
-    # Filter out unsupported configurations on OS X.
-    if os == 'osx' and ('dss:primary' in malloc_conf or \
-      'percpu_arena:percpu' in malloc_conf or 'background_thread:true' \
-      in malloc_conf):
-        return ""
+# Formats a job from a combination of flags
+def format_job(os, arch, combination):
+    compilers = [x.value for x in combination if x.type == Option.Type.COMPILER]
+    assert(len(compilers) <= 1)
+    compiler_flags = [x.value for x in combination if x.type == Option.Type.COMPILER_FLAG]
+    configure_flags = [x.value for x in combination if x.type == Option.Type.CONFIGURE_FLAG]
+    malloc_conf = [x.value for x in combination if x.type == Option.Type.MALLOC_CONF]
+    features = [x.value for x in combination if x.type == Option.Type.FEATURE]
+
     if len(malloc_conf) > 0:
-        configure_flags.append('--with-malloc-conf=' + ",".join(malloc_conf))
+        configure_flags.append('--with-malloc-conf=' + ','.join(malloc_conf))
 
-    # Filter out an unsupported configuration - heap profiling on OS X.
-    if os == 'osx' and '--enable-prof' in configure_flags:
-        return ""
+    if not compilers:
+        compiler = GCC.value
+    else:
+        compiler = compilers[0]
 
-    # We get some spurious errors when -Warray-bounds is enabled.
-    env_string = ('{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}" '
-	'EXTRA_CFLAGS="-Werror -Wno-array-bounds"').format(
-        compilers, " ".join(compiler_flags), " ".join(configure_flags))
-
-    job = ""
-    job += '    - os: %s\n' % os
-    job += '      env: %s\n' % env_string
-    if '-m32' in combination and os == 'linux':
-        job += '      addons:'
-        if gcc_multilib_set:
-            job += ' *gcc_multilib\n'
-        else:
-            job += ' &gcc_multilib\n'
-            job += '        apt:\n'
-            job += '          packages:\n'
-            job += '            - gcc-multilib\n'
-            gcc_multilib_set = True
+    extra_environment_vars = ''
+    cross_compile = CROSS_COMPILE_32BIT.value in features
+    if os == LINUX and cross_compile:
+        compiler_flags.append('-m32')
+
+    features_str = ' '.join([' {}=yes'.format(feature) for feature in features])
+
+    stringify = lambda arr, name: ' {}="{}"'.format(name, ' '.join(arr)) if arr else ''
+    env_string = '{}{}{}{}{}{}'.format(
+            compiler,
+            features_str,
+            stringify(compiler_flags, 'COMPILER_FLAGS'),
+            stringify(configure_flags, 'CONFIGURE_FLAGS'),
+            stringify(get_extra_cflags(os, compiler), 'EXTRA_CFLAGS'),
+            extra_environment_vars)
+
+    job = '    - os: {}\n'.format(os)
+    job += '      arch: {}\n'.format(arch)
+    job += '      env: {}'.format(env_string)
     return job
 
-include_rows = ""
-for combination in unusual_combinations_to_test:
-    include_rows += format_job(combination)
 
-# Development build
-include_rows += '''\
+def generate_unusual_combinations(unusuals, max_unusual_opts):
+    """
+    Generates different combinations of non-standard compilers, compiler flags,
+    configure flags and malloc_conf settings.
+
+    @param max_unusual_opts: Limit of unusual options per combination.
+    """
+    return chain.from_iterable(
+            [combinations(unusuals, i) for i in range(max_unusual_opts + 1)])
+
+
+def included(combination, exclude):
+    """
+    Checks if the combination of options should be included in the Travis
+    testing matrix.
+
+    @param exclude: A list of options to be avoided.
+    """
+    return not any(excluded in combination for excluded in exclude)
+
+
+def generate_jobs(os, arch, exclude, max_unusual_opts, unusuals=all_unusuals):
+    jobs = []
+    for combination in generate_unusual_combinations(unusuals, max_unusual_opts):
+        if included(combination, exclude):
+            jobs.append(format_job(os, arch, combination))
+    return '\n'.join(jobs)
+
+
+def generate_linux(arch):
+    os = LINUX
+
+    # Only generate 2 unusual options for AMD64 to reduce matrix size
+    max_unusual_opts = MAX_UNUSUAL_OPTIONS if arch == AMD64 else 1
+
+    exclude = []
+    if arch == PPC64LE:
+        # Avoid 32 bit builds and clang on PowerPC
+        exclude = (CROSS_COMPILE_32BIT, CLANG,)
+
+    return generate_jobs(os, arch, exclude, max_unusual_opts)
+
+
+def generate_macos(arch):
+    os = OSX
+
+    max_unusual_opts = 1
+
+    exclude = ([Option.as_malloc_conf(opt) for opt in (
+            'dss:primary',
+            'percpu_arena:percpu',
+            'background_thread:true')] +
+        [Option.as_configure_flag('--enable-prof')] +
+        [CLANG,])
+
+    return generate_jobs(os, arch, exclude, max_unusual_opts)
+
+
+def generate_windows(arch):
+    os = WINDOWS
+
+    max_unusual_opts = 3
+    unusuals = (
+        Option.as_configure_flag('--enable-debug'),
+        CL,
+        CROSS_COMPILE_32BIT,
+    )
+    return generate_jobs(os, arch, (), max_unusual_opts, unusuals)
+
+
+def generate_freebsd(arch):
+    os = FREEBSD
+
+    max_unusual_opts = 4
+    unusuals = (
+        Option.as_configure_flag('--enable-debug'),
+        Option.as_configure_flag('--enable-prof --enable-prof-libunwind'),
+        Option.as_configure_flag('--with-lg-page=16 --with-malloc-conf=tcache:false'),
+        CROSS_COMPILE_32BIT,
+    )
+    return generate_jobs(os, arch, (), max_unusual_opts, unusuals)
+
+
+
+def get_manual_jobs():
+    return """\
     # Development build
     - os: linux
-      env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
-'''
-
-# Enable-expermental-smallocx
-include_rows += '''\
+      env: CC=gcc CXX=g++ CONFIGURE_FLAGS="--enable-debug \
+--disable-cache-oblivious --enable-stats --enable-log --enable-prof" \
+EXTRA_CFLAGS="-Werror -Wno-array-bounds"
     # --enable-expermental-smallocx:
     - os: linux
-      env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
-'''
+      env: CC=gcc CXX=g++ CONFIGURE_FLAGS="--enable-debug \
+--enable-experimental-smallocx --enable-stats --enable-prof" \
+EXTRA_CFLAGS="-Werror -Wno-array-bounds"
+"""
 
-# Valgrind build bots
-include_rows += '''
-    # Valgrind
-    - os: linux
-      env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind"
-      addons:
-        apt:
-          packages:
-            - valgrind
-'''
-
-# To enable valgrind on macosx add:
-#
-#  - os: osx
-#    env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind"
-#    install: brew install valgrind
-#
-# It currently fails due to: https://github.com/jemalloc/jemalloc/issues/1274
-
-print travis_template % include_rows
+
+def main():
+    jobs = '\n'.join((
+        generate_windows(AMD64),
+
+        generate_freebsd(AMD64),
+
+        generate_linux(AMD64),
+        generate_linux(PPC64LE),
+
+        generate_macos(AMD64),
+
+        get_manual_jobs(),
+    ))
+
+    print(TRAVIS_TEMPLATE.format(jobs=jobs))
+
+
+if __name__ == '__main__':
+    main()