diff options
Diffstat (limited to 'tools')
-rw-r--r-- | tools/allocation_tracking/README.md | 7 | ||||
-rw-r--r-- | tools/ci/cirrus_general.yml | 99 | ||||
-rw-r--r-- | tools/ci/cirrus_macosx_arm64.yml | 55 | ||||
-rw-r--r-- | tools/ci/cirrus_wheels.yml | 157 | ||||
-rwxr-xr-x | tools/ci/push_docs_to_repo.py | 7 | ||||
-rwxr-xr-x | tools/cythonize.py | 10 | ||||
-rw-r--r-- | tools/gitpod/Dockerfile | 101 | ||||
-rw-r--r-- | tools/gitpod/gitpod.Dockerfile | 49 | ||||
-rw-r--r-- | tools/gitpod/settings.json | 8 | ||||
-rw-r--r-- | tools/gitpod/workspace_config | 58 | ||||
-rw-r--r-- | tools/list_installed_dll_dependencies_cygwin.sh | 10 | ||||
-rw-r--r-- | tools/openblas_support.py | 60 | ||||
-rw-r--r-- | tools/rebase_installed_dlls_cygwin.sh | 4 | ||||
-rw-r--r-- | tools/refguide_check.py | 6 | ||||
-rw-r--r-- | tools/swig/pyfragments.swg | 2 | ||||
-rwxr-xr-x | tools/travis-test.sh | 2 | ||||
-rw-r--r-- | tools/wheels/check_license.py | 3 | ||||
-rw-r--r-- | tools/wheels/cibw_before_build.sh | 12 | ||||
-rw-r--r-- | tools/wheels/gfortran_utils.sh | 6 | ||||
-rw-r--r-- | tools/wheels/upload_wheels.sh | 2 |
20 files changed, 291 insertions, 367 deletions
diff --git a/tools/allocation_tracking/README.md b/tools/allocation_tracking/README.md deleted file mode 100644 index 6cc4c2a58..000000000 --- a/tools/allocation_tracking/README.md +++ /dev/null @@ -1,7 +0,0 @@ -Note that since Python 3.6 the builtin tracemalloc module can be used to -track allocations inside numpy. -Numpy places its CPU memory allocations into the `np.lib.tracemalloc_domain` -domain. -See https://docs.python.org/3/library/tracemalloc.html. - -The tool that used to be here has been deprecated. diff --git a/tools/ci/cirrus_general.yml b/tools/ci/cirrus_general.yml deleted file mode 100644 index 2ec2ad837..000000000 --- a/tools/ci/cirrus_general.yml +++ /dev/null @@ -1,99 +0,0 @@ -build_and_store_wheels: &BUILD_AND_STORE_WHEELS - install_cibuildwheel_script: - - python -m pip install cibuildwheel==2.11.2 - cibuildwheel_script: - - cibuildwheel - wheels_artifacts: - path: "wheelhouse/*" - - -###################################################################### -# Build linux_aarch64 natively -###################################################################### - -linux_aarch64_task: - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder-arm64 - architecture: arm64 - platform: linux - cpu: 2 - memory: 8G - matrix: - # build in a matrix because building and testing all four wheels in a - # single task takes longer than 60 mins (the default time limit for a - # cirrus-ci task). - - env: - CIBW_BUILD: cp38-* - EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM - - env: - CIBW_BUILD: cp39-* - - env: - CIBW_BUILD: cp310-* - - env: - CIBW_BUILD: cp311-* - - build_script: | - apt install -y python3-venv python-is-python3 gfortran libatlas-base-dev libgfortran5 eatmydata - git fetch origin - ./tools/travis-before-install.sh - which python - echo $CIRRUS_CHANGE_MESSAGE - <<: *BUILD_AND_STORE_WHEELS - - -###################################################################### -# Upload all wheels -###################################################################### - -wheels_upload_task: - # Artifacts don't seem to be persistent from task to task. - # Rather than upload wheels at the end of each cibuildwheel run we do a - # final upload here. This is because a run may be on different OS for - # which bash, etc, may not be present. - depends_on: - - linux_aarch64 - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder - platform: linux - cpu: 1 - - env: - NUMPY_STAGING_UPLOAD_TOKEN: ENCRYPTED[!5a69522ae0c2af9edb2bc1cdfeaca6292fb3666d9ecd82dca0615921834a6ce3b702352835d8bde4ea2a9ed5ef8424ac!] - NUMPY_NIGHTLY_UPLOAD_TOKEN: ENCRYPTED[!196422e6c3419a3b1d79815e1026094a215cb0f346fe34ed0f9d3ca1c19339df7398d04556491b1e0420fc1fe3713289!] - - upload_script: | - apt-get install -y python3-venv python-is-python3 curl - export IS_SCHEDULE_DISPATCH="false" - export IS_PUSH="false" - - # cron job - if [[ "$CIRRUS_CRON" == "weekly" ]]; then - export IS_SCHEDULE_DISPATCH="true" - fi - - # a manual build was started - if [[ "$CIRRUS_BUILD_SOURCE" == "api" && "$CIRRUS_COMMIT_MESSAGE" == "API build for null" ]]; then - export IS_SCHEDULE_DISPATCH="true" - fi - - # only upload wheels to staging if it's a tag beginning with 'v' and you're - # on a maintenance branch - if [[ "$CIRRUS_TAG" == v* ]] && [[ "$CIRRUS_BRANCH" == maintenance* ]]; then - export IS_PUSH="true" - fi - - # The name of the zip file is derived from the `wheels_artifact` line. - # If you change the artifact line to `myfile_artifact` then it would be - # called myfile.zip - - curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/wheels.zip --output wheels.zip - unzip wheels.zip - - source ./tools/wheels/upload_wheels.sh - # IS_PUSH takes precedence over IS_SCHEDULE_DISPATCH - set_upload_vars - - # Will be skipped if not a push/tag/scheduled build - upload_wheels diff --git a/tools/ci/cirrus_macosx_arm64.yml b/tools/ci/cirrus_macosx_arm64.yml new file mode 100644 index 000000000..2343b807f --- /dev/null +++ b/tools/ci/cirrus_macosx_arm64.yml @@ -0,0 +1,55 @@ +modified_clone: &MODIFIED_CLONE + # makes sure that for a PR the CI runs against a merged main + clone_script: | + if [ -z "$CIRRUS_PR" ]; then + # if you're not in a PR then clone against the branch name that was pushed to. + git clone --recursive --branch=$CIRRUS_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR + git reset --hard $CIRRUS_CHANGE_IN_REPO + else + # it's a PR so clone the main branch then merge the changes from the PR + git clone https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR + git fetch origin pull/$CIRRUS_PR/head:pull/$CIRRUS_PR + + # CIRRUS_BASE_BRANCH will probably be `main` for the majority of the time + # However, if you do a PR against a maintenance branch we will want to + # merge the PR into the maintenance branch, not main + git checkout $CIRRUS_BASE_BRANCH + + # alpine git package needs default user.name and user.email to be set before a merge + git -c user.email="you@example.com" merge --no-commit pull/$CIRRUS_PR + git submodule update --init --recursive + fi + + +macos_arm64_test_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:14 + + <<: *MODIFIED_CLONE + + pip_cache: + folder: ~/.cache/pip + + test_script: | + brew install python@3.10 + + export PATH=/opt/homebrew/opt/python@3.10/libexec/bin:$PATH + python --version + + RUNNER_OS="macOS" + CFLAGS="-std=c99 -fno-strict-aliasing" + SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk + + # used for installing OpenBLAS/gfortran + bash tools/wheels/cibw_before_build.sh $PWD + + pushd ~/ + python -m venv numpy-dev + source numpy-dev/bin/activate + popd + + pip install -r build_requirements.txt + pip install pytest hypothesis typing_extensions + + spin build + spin test diff --git a/tools/ci/cirrus_wheels.yml b/tools/ci/cirrus_wheels.yml new file mode 100644 index 000000000..60512afab --- /dev/null +++ b/tools/ci/cirrus_wheels.yml @@ -0,0 +1,157 @@ +build_and_store_wheels: &BUILD_AND_STORE_WHEELS + install_cibuildwheel_script: + - python -m pip install cibuildwheel==2.12.1 + cibuildwheel_script: + - cibuildwheel + wheels_artifacts: + path: "wheelhouse/*" + +###################################################################### +# Build linux_aarch64 natively +###################################################################### + +linux_aarch64_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 2 + memory: 8G + matrix: + # build in a matrix because building and testing all four wheels in a + # single task takes longer than 60 mins (the default time limit for a + # cirrus-ci task). + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp39-* + EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp310-* + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp311-* + + build_script: | + apt install -y python3-venv python-is-python3 gfortran libatlas-base-dev libgfortran5 eatmydata + git fetch origin + ./tools/travis-before-install.sh + which python + echo $CIRRUS_CHANGE_MESSAGE + <<: *BUILD_AND_STORE_WHEELS + + +###################################################################### +# Build macosx_arm64 natively +###################################################################### + +macosx_arm64_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:14 + matrix: + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp39-* + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp310-* cp311-* + env: + PATH: /opt/homebrew/opt/python@3.10/bin:/usr/local/lib:/usr/local/include:$PATH + CIBW_ARCHS: arm64 + # Specifying CIBW_ENVIRONMENT_MACOS overrides pyproject.toml, so include + # all the settings from there, otherwise they're lost. + # SDKROOT needs to be set for repackaged conda-forge gfortran compilers + # supplied by isuruf. + # Find out SDKROOT via `xcrun --sdk macosx --show-sdk-path` + CIBW_ENVIRONMENT_MACOS: > + RUNNER_OS=macOS + SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk + LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + CFLAGS="-std=c99 -fno-strict-aliasing" + OPENBLAS64_="/usr/local" + NPY_USE_BLAS_ILP64="1" + + build_script: + - brew install python@3.10 + - ln -s python3 /opt/homebrew/opt/python@3.10/bin/python + - which python + # needed for submodules + - git submodule update --init + # need to obtain all the tags so setup.py can determine FULLVERSION + - git fetch origin + - uname -m + - python -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" + - clang --version + <<: *BUILD_AND_STORE_WHEELS + + +###################################################################### +# Upload all wheels +###################################################################### + +wheels_upload_task: + # Artifacts don't seem to be persistent from task to task. + # Rather than upload wheels at the end of each cibuildwheel run we do a + # final upload here. This is because a run may be on different OS for + # which bash, etc, may not be present. + depends_on: + - linux_aarch64 + - macosx_arm64 + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + + env: + NUMPY_STAGING_UPLOAD_TOKEN: ENCRYPTED[!5a69522ae0c2af9edb2bc1cdfeaca6292fb3666d9ecd82dca0615921834a6ce3b702352835d8bde4ea2a9ed5ef8424ac!] + NUMPY_NIGHTLY_UPLOAD_TOKEN: ENCRYPTED[!196422e6c3419a3b1d79815e1026094a215cb0f346fe34ed0f9d3ca1c19339df7398d04556491b1e0420fc1fe3713289!] + + upload_script: | + apt-get update + apt-get install -y curl wget + export IS_SCHEDULE_DISPATCH="false" + export IS_PUSH="false" + + # cron job + if [[ "$CIRRUS_CRON" == "weekly" ]]; then + export IS_SCHEDULE_DISPATCH="true" + fi + + # a manual build was started + if [[ "$CIRRUS_BUILD_SOURCE" == "api" && "$CIRRUS_COMMIT_MESSAGE" == "API build for null" ]]; then + export IS_SCHEDULE_DISPATCH="true" + fi + + # only upload wheels to staging if it's a tag beginning with 'v' and you're + # on a maintenance branch + if [[ "$CIRRUS_TAG" == v* ]] && [[ $CIRRUS_TAG != *"dev0"* ]]; then + export IS_PUSH="true" + fi + + if [[ $IS_PUSH == "true" ]] || [[ $IS_SCHEDULE_DISPATCH == "true" ]]; then + # install miniconda in the home directory. For some reason HOME isn't set by Cirrus + export HOME=$PWD + + # install miniconda for uploading to anaconda + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda3 + $HOME/miniconda3/bin/conda init bash + source $HOME/miniconda3/bin/activate + conda install -y anaconda-client + + # The name of the zip file is derived from the `wheels_artifact` line. + # If you change the artifact line to `myfile_artifact` then it would be + # called myfile.zip + + curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/wheels.zip --output wheels.zip + unzip wheels.zip + + source ./tools/wheels/upload_wheels.sh + # IS_PUSH takes precedence over IS_SCHEDULE_DISPATCH + set_upload_vars + + # Will be skipped if not a push/tag/scheduled build + upload_wheels + fi diff --git a/tools/ci/push_docs_to_repo.py b/tools/ci/push_docs_to_repo.py index e34522a29..0471e3824 100755 --- a/tools/ci/push_docs_to_repo.py +++ b/tools/ci/push_docs_to_repo.py @@ -19,6 +19,8 @@ parser.add_argument('--committer', default='numpy-commit-bot', help='Name of the git committer') parser.add_argument('--email', default='numpy-commit-bot@nomail', help='Email of the git committer') +parser.add_argument('--count', default=1, type=int, + help="minimum number of expected files, defaults to 1") parser.add_argument( '--force', action='store_true', @@ -31,6 +33,11 @@ if not os.path.exists(args.dir): print('Content directory does not exist') sys.exit(1) +count = len([name for name in os.listdir(args.dir) if os.path.isfile(os.path.join(args.dir, name))]) + +if count < args.count: + print(f"Expected {args.count} top-directory files to upload, got {count}") + sys.exit(1) def run(cmd, stdout=True): pipe = None if stdout else subprocess.DEVNULL diff --git a/tools/cythonize.py b/tools/cythonize.py index 311a52c91..d66ddb98c 100755 --- a/tools/cythonize.py +++ b/tools/cythonize.py @@ -52,7 +52,7 @@ def process_tempita_pyx(fromfile, tofile): import npy_tempita as tempita assert fromfile.endswith('.pyx.in') - with open(fromfile, "r") as f: + with open(fromfile) as f: tmpl = f.read() pyxcontent = tempita.sub(tmpl) pyxfile = fromfile[:-len('.pyx.in')] + '.pyx' @@ -66,7 +66,7 @@ def process_tempita_pyd(fromfile, tofile): assert fromfile.endswith('.pxd.in') assert tofile.endswith('.pxd') - with open(fromfile, "r") as f: + with open(fromfile) as f: tmpl = f.read() pyxcontent = tempita.sub(tmpl) with open(tofile, "w") as f: @@ -77,7 +77,7 @@ def process_tempita_pxi(fromfile, tofile): assert fromfile.endswith('.pxi.in') assert tofile.endswith('.pxi') - with open(fromfile, "r") as f: + with open(fromfile) as f: tmpl = f.read() pyxcontent = tempita.sub(tmpl) with open(tofile, "w") as f: @@ -88,7 +88,7 @@ def process_tempita_pxd(fromfile, tofile): assert fromfile.endswith('.pxd.in') assert tofile.endswith('.pxd') - with open(fromfile, "r") as f: + with open(fromfile) as f: tmpl = f.read() pyxcontent = tempita.sub(tmpl) with open(tofile, "w") as f: @@ -109,7 +109,7 @@ def load_hashes(filename): # Return { filename : (sha256 of input, sha256 of output) } if os.path.isfile(filename): hashes = {} - with open(filename, 'r') as f: + with open(filename) as f: for line in f: filename, inhash, outhash = line.split() hashes[filename] = (inhash, outhash) diff --git a/tools/gitpod/Dockerfile b/tools/gitpod/Dockerfile deleted file mode 100644 index dd5561750..000000000 --- a/tools/gitpod/Dockerfile +++ /dev/null @@ -1,101 +0,0 @@ -# -# Dockerfile for NumPy development -# -# Usage: -# ------- -# -# To make a local build of the container, from the 'Docker-dev' directory: -# docker build --rm -f "Dockerfile" -t <build-tag> "." -# -# To use the container use the following command. It assumes that you are in -# the root folder of the NumPy git repository, making it available as -# /home/numpy in the container. Whatever changes you make to that directory -# are visible in the host and container. -# The docker image is retrieved from the NumPy dockerhub repository -# -# docker run --rm -it -v $(pwd):/home/numpy numpy/numpy-dev:<image-tag> -# -# By default the container will activate the conda environment numpy-dev -# which contains all the dependencies needed for NumPy development -# -# To build NumPy run: python setup.py build_ext --inplace -# -# To run the tests use: python runtests.py -# -# This image is based on: Ubuntu 20.04 (focal) -# https://hub.docker.com/_/ubuntu/?tab=tags&name=focal -# OS/ARCH: linux/amd64 -FROM gitpod/workspace-base:latest - -ARG MAMBAFORGE_VERSION="4.11.0-0" -ARG CONDA_ENV=numpy-dev - - -# ---- Configure environment ---- -ENV CONDA_DIR=/home/gitpod/mambaforge3 \ - SHELL=/bin/bash -ENV PATH=${CONDA_DIR}/bin:$PATH \ - WORKSPACE=/workspace/numpy - - -# ----------------------------------------------------------------------------- -# ---- Creating as root - note: make sure to change to gitpod in the end ---- -USER root - -# hadolint ignore=DL3008 -RUN apt-get update && \ - apt-get install -yq --no-install-recommends \ - ca-certificates \ - dirmngr \ - dvisvgm \ - gnupg \ - gpg-agent \ - texlive-latex-extra \ - vim && \ - # this needs to be done after installing dirmngr - apt-key adv --keyserver keyserver.ubuntu.com --recv-key 23F3D4EA75716059 && \ - apt-add-repository https://cli.github.com/packages && \ - apt-get install -yq --no-install-recommends \ - gh && \ - locale-gen en_US.UTF-8 && \ - apt-get clean && \ - rm -rf /var/cache/apt/* &&\ - rm -rf /var/lib/apt/lists/* &&\ - rm -rf /tmp/* - -# Allows this Dockerfile to activate conda environments -SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] - -# ----------------------------------------------------------------------------- -# ---- Installing mamba ---- -RUN wget -q -O mambaforge3.sh \ - "https://github.com/conda-forge/miniforge/releases/download/$MAMBAFORGE_VERSION/Mambaforge-$MAMBAFORGE_VERSION-Linux-x86_64.sh" && \ - bash mambaforge3.sh -p ${CONDA_DIR} -b && \ - rm mambaforge3.sh - -# ----------------------------------------------------------------------------- -# ---- Copy needed files ---- -# basic workspace configurations -COPY ./tools/gitpod/workspace_config /usr/local/bin/workspace_config - -RUN chmod a+rx /usr/local/bin/workspace_config && \ - workspace_config - -# Copy conda environment file into the container - this needs to exists inside -# the container to create a conda environment from it -COPY environment.yml /tmp/environment.yml - -# ----------------------------------------------------------------------------- -# ---- Create conda environment ---- -# Install NumPy dependencies -RUN mamba env create -f /tmp/environment.yml && \ - conda activate ${CONDA_ENV} && \ - mamba install ccache -y && \ - # needed for docs rendering later on - python -m pip install --no-cache-dir sphinx-autobuild && \ - conda clean --all -f -y && \ - rm -rf /tmp/* - -# ----------------------------------------------------------------------------- -# Always make sure we are not root -USER gitpod
\ No newline at end of file diff --git a/tools/gitpod/gitpod.Dockerfile b/tools/gitpod/gitpod.Dockerfile deleted file mode 100644 index 8dac0d597..000000000 --- a/tools/gitpod/gitpod.Dockerfile +++ /dev/null @@ -1,49 +0,0 @@ -# Doing a local shallow clone - keeps the container secure -# and much slimmer than using COPY directly or making a -# remote clone -ARG BASE_CONTAINER="numpy/numpy-dev:latest" -FROM gitpod/workspace-base:latest as clone - -COPY --chown=gitpod . /tmp/numpy_repo - -# the clone should be deep enough for versioneer to work -RUN git clone --shallow-since=2021-05-22 file:////tmp/numpy_repo /tmp/numpy - -# ----------------------------------------------------------------------------- -# Using the numpy-dev Docker image as a base -# This way, we ensure we have all the needed compilers and dependencies -# while reducing the build time -FROM ${BASE_CONTAINER} as build - -# ----------------------------------------------------------------------------- -USER root - -# ----------------------------------------------------------------------------- -# ---- ENV variables ---- -# ---- Directories needed ---- -ENV WORKSPACE=/workspace/numpy/ \ - CONDA_ENV=numpy-dev - -# Allows this Dockerfile to activate conda environments -SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] - -# Copy over the shallow clone -COPY --from=clone --chown=gitpod /tmp/numpy ${WORKSPACE} - -# Everything happens in the /workspace/numpy directory -WORKDIR ${WORKSPACE} - -# Build numpy to populate the cache used by ccache -RUN git config --global --add safe.directory /workspace/numpy -RUN git submodule update --init --depth=1 -- numpy/core/src/umath/svml -RUN conda activate ${CONDA_ENV} && \ - python setup.py build_ext --inplace && \ - ccache -s - -# Gitpod will load the repository into /workspace/numpy. We remove the -# directory from the image to prevent conflicts -RUN rm -rf ${WORKSPACE} - -# ----------------------------------------------------------------------------- -# Always return to non privileged user -USER gitpod diff --git a/tools/gitpod/settings.json b/tools/gitpod/settings.json deleted file mode 100644 index 50296336d..000000000 --- a/tools/gitpod/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "restructuredtext.updateOnTextChanged": "true", - "restructuredtext.updateDelay": 300, - "restructuredtext.linter.disabledLinters": ["doc8","rst-lint", "rstcheck"], - "python.defaultInterpreterPath": "/home/gitpod/mambaforge3/envs/numpy-dev/bin/python", - "esbonio.sphinx.buildDir": "${workspaceRoot}/doc/build/html", - "esbonio.sphinx.confDir": "" -}
\ No newline at end of file diff --git a/tools/gitpod/workspace_config b/tools/gitpod/workspace_config deleted file mode 100644 index aa859c9be..000000000 --- a/tools/gitpod/workspace_config +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -# Basic configurations for the workspace - -set -e - -# gitpod/workspace-base needs at least one file here -touch /home/gitpod/.bashrc.d/empty - -# Add git aliases -git config --global alias.co checkout -git config --global alias.ci commit -git config --global alias.st status -git config --global alias.br branch -git config --global alias.hist "log --pretty=format:'%h %ad | %s%d [%an]' --graph --date=short" -git config --global alias.type 'cat-file -t' -git config --global alias.dump 'cat-file -p' - -# Enable basic vim defaults in ~/.vimrc -echo "filetype plugin indent on" >>~/.vimrc -echo "set colorcolumn=80" >>~/.vimrc -echo "set number" >>~/.vimrc -echo "syntax enable" >>~/.vimrc - -# Vanity custom bash prompt - makes it more legible -echo "PS1='\[\e]0;\u \w\a\]\[\033[01;36m\]\u\[\033[m\] > \[\033[38;5;141m\]\w\[\033[m\] \\$ '" >>~/.bashrc - -# Enable prompt color in the skeleton .bashrc -# hadolint ignore=SC2016 -sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc - -# .gitpod.yml is configured to install NumPy from /workspace/numpy -echo "export PYTHONPATH=${WORKSPACE}" >>~/.bashrc - -# make conda activate command available from /bin/bash (login and interactive) -if [[ ! -f "/etc/profile.d/conda.sh" ]]; then - ln -s ${CONDA_DIR}/etc/profile.d/conda.sh /etc/profile.d/conda.sh -fi -echo ". ${CONDA_DIR}/etc/profile.d/conda.sh" >>~/.bashrc -echo "conda activate numpy-dev" >>~/.bashrc - -# Enable prompt color in the skeleton .bashrc -# hadolint ignore=SC2016 -sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc - -# .gitpod.yml is configured to install numpy from /workspace/numpy -echo "export PYTHONPATH=/workspace/numpy" >>~/.bashrc - -# Set up ccache for compilers for this Dockerfile -# REF: https://github.com/conda-forge/compilers-feedstock/issues/31 -echo "conda activate numpy-dev" >>~/.startuprc -echo "export CC=\"ccache \$CC\"" >>~/.startuprc -echo "export CXX=\"ccache \$CXX\"" >>~/.startuprc -echo "export F77=\"ccache \$F77\"" >>~/.startuprc -echo "export F90=\"ccache \$F90\"" >>~/.startuprc -echo "export GFORTRAN=\"ccache \$GFORTRAN\"" >>~/.startuprc -echo "export FC=\"ccache \$FC\"" >>~/.startuprc -echo "source ~/.startuprc" >>~/.profile -echo "source ~/.startuprc" >>~/.bashrc diff --git a/tools/list_installed_dll_dependencies_cygwin.sh b/tools/list_installed_dll_dependencies_cygwin.sh index ee06ae0d0..78436fe53 100644 --- a/tools/list_installed_dll_dependencies_cygwin.sh +++ b/tools/list_installed_dll_dependencies_cygwin.sh @@ -14,11 +14,11 @@ py_ver=${1} dll_list=`/bin/dash tools/list_numpy_dlls.sh ${py_ver}` echo "Checks for existence, permissions and file type" -ls -l ${dll_list} -file ${dll_list} +/usr/bin/timeout 10m /usr/bin/ls -l ${dll_list} +/usr/bin/timeout 10m /usr/bin/file ${dll_list} echo "Dependency checks" -ldd ${dll_list} | grep -F -e " => not found" && exit 1 -cygcheck ${dll_list} >cygcheck_dll_list 2>cygcheck_missing_deps +/usr/bin/timeout 10m /usr/bin/ldd ${dll_list} | grep -F -e " => not found" && exit 1 +/usr/bin/timeout 10m /usr/bin/cygcheck ${dll_list} >cygcheck_dll_list 2>cygcheck_missing_deps grep -F -e "cygcheck: track_down: could not find " cygcheck_missing_deps && exit 1 echo "Import tests" mkdir -p dist/ @@ -31,5 +31,5 @@ do -e "s/^\/+(home|usr).*?site-packages\/+//" \ -e "s/.cpython-3.m?-x86(_64)?-cygwin.dll$//" \ -e "s/\//./g"` - python${py_ver} -c "import ${ext_module}" + /usr/bin/timeout 2m /usr/bin/python${py_ver} -c "import ${ext_module}" done diff --git a/tools/openblas_support.py b/tools/openblas_support.py index ed43804af..fd8c6a97a 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -13,13 +13,14 @@ from tempfile import mkstemp, gettempdir from urllib.request import urlopen, Request from urllib.error import HTTPError -OPENBLAS_V = '0.3.21' -OPENBLAS_LONG = 'v0.3.21' +OPENBLAS_V = '0.3.23' +OPENBLAS_LONG = 'v0.3.23' BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs' BASEURL = f'{BASE_LOC}/{OPENBLAS_LONG}/download' SUPPORTED_PLATFORMS = [ 'linux-aarch64', 'linux-x86_64', + 'musllinux-x86_64', 'linux-i686', 'linux-ppc64le', 'linux-s390x', @@ -55,10 +56,39 @@ def get_ilp64(): def get_manylinux(arch): default = '2014' - ret = os.environ.get("MB_ML_VER", default) + ml_ver = os.environ.get("MB_ML_VER", default) # XXX For PEP 600 this can be a glibc version - assert ret in ('2010', '2014', '_2_24'), f'invalid MB_ML_VER {ret}' - return ret + assert ml_ver in ('2010', '2014', '_2_24'), f'invalid MB_ML_VER {ml_ver}' + suffix = f'manylinux{ml_ver}_{arch}.tar.gz' + return suffix + + +def get_musllinux(arch): + musl_ver = "1_1" + suffix = f'musllinux_{musl_ver}_{arch}.tar.gz' + return suffix + + +def get_linux(arch): + # best way of figuring out whether manylinux or musllinux is to look + # at the packaging tags. If packaging isn't installed (it's not by default) + # fallback to sysconfig (which may be flakier) + try: + from packaging.tags import sys_tags + tags = list(sys_tags()) + plat = tags[0].platform + except ImportError: + # fallback to sysconfig for figuring out if you're using musl + plat = 'manylinux' + # value could be None + v = sysconfig.get_config_var('HOST_GNU_TYPE') or '' + if 'musl' in v: + plat = 'musllinux' + + if 'manylinux' in plat: + return get_manylinux(arch) + elif 'musllinux' in plat: + return get_musllinux(arch) def download_openblas(target, plat, ilp64): @@ -70,8 +100,10 @@ def download_openblas(target, plat, ilp64): '(KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3')} suffix = None if osname == "linux": - ml_ver = get_manylinux(arch) - suffix = f'manylinux{ml_ver}_{arch}.tar.gz' + suffix = get_linux(arch) + typ = 'tar.gz' + elif osname == "musllinux": + suffix = get_musllinux(arch) typ = 'tar.gz' elif plat == 'macosx-x86_64': suffix = 'macosx_10_9_x86_64-gf_c469a42.tar.gz' @@ -200,7 +232,7 @@ def make_init(dirname): ''' Create a _distributor_init.py file for OpenBlas ''' - with open(os.path.join(dirname, '_distributor_init.py'), 'wt') as fid: + with open(os.path.join(dirname, '_distributor_init.py'), 'w') as fid: fid.write(textwrap.dedent(""" ''' Helper to preload windows dlls to prevent dll not found errors. @@ -267,15 +299,11 @@ def test_setup(plats): if not target: raise RuntimeError(f'Could not setup {plat}') print('success with', plat, ilp64) - if osname == 'win': - if not target.endswith('.a'): - raise RuntimeError("Not .a extracted!") - else: - files = glob.glob(os.path.join(target, "lib", "*.a")) - if not files: - raise RuntimeError("No lib/*.a unpacked!") + files = glob.glob(os.path.join(target, "lib", "*.a")) + if not files: + raise RuntimeError("No lib/*.a unpacked!") finally: - if target is not None: + if target: if os.path.isfile(target): os.unlink(target) else: diff --git a/tools/rebase_installed_dlls_cygwin.sh b/tools/rebase_installed_dlls_cygwin.sh index f772879d9..58d4f0c0f 100644 --- a/tools/rebase_installed_dlls_cygwin.sh +++ b/tools/rebase_installed_dlls_cygwin.sh @@ -2,4 +2,6 @@ # Rebase the dlls installed by NumPy py_ver=${1} -/usr/bin/rebase --database --oblivious `/bin/dash tools/list_numpy_dlls.sh ${py_ver}` +numpy_dlls="`/bin/dash tools/list_numpy_dlls.sh ${py_ver}`" +/usr/bin/rebase --verbose --database --oblivious ${numpy_dlls} +/usr/bin/rebase --verbose --info ${numpy_dlls} diff --git a/tools/refguide_check.py b/tools/refguide_check.py index 1d230b385..2c69b9464 100644 --- a/tools/refguide_check.py +++ b/tools/refguide_check.py @@ -99,12 +99,6 @@ DOCTEST_SKIPDICT = { 'numpy.lib.DataSource': None, 'numpy.lib.Repository': None, } -if sys.version_info < (3, 9): - DOCTEST_SKIPDICT.update({ - "numpy.core.ndarray": {"__class_getitem__"}, - "numpy.core.dtype": {"__class_getitem__"}, - "numpy.core.number": {"__class_getitem__"}, - }) # Skip non-numpy RST files, historical release notes # Any single-directory exact match will skip the directory and all subdirs. diff --git a/tools/swig/pyfragments.swg b/tools/swig/pyfragments.swg index eac817322..6d3e6ff41 100644 --- a/tools/swig/pyfragments.swg +++ b/tools/swig/pyfragments.swg @@ -52,6 +52,7 @@ } %#endif if (!PyArray_IsScalar(obj,Integer)) return SWIG_TypeError; + if (!val) return SWIG_OK; PyArray_Descr * longDescr = PyArray_DescrFromType(NPY_LONG); PyArray_CastScalarToCtype(obj, (void*)val, longDescr); Py_DECREF(longDescr); @@ -102,6 +103,7 @@ } %#endif if (!PyArray_IsScalar(obj,Integer)) return SWIG_TypeError; + if (!val) return SWIG_OK; PyArray_Descr * ulongDescr = PyArray_DescrFromType(NPY_ULONG); PyArray_CastScalarToCtype(obj, (void*)val, ulongDescr); Py_DECREF(ulongDescr); diff --git a/tools/travis-test.sh b/tools/travis-test.sh index 6216c239b..67e507eb3 100755 --- a/tools/travis-test.sh +++ b/tools/travis-test.sh @@ -33,7 +33,7 @@ werrors="$werrors -Werror=implicit-function-declaration" setup_base() { # use default python flags but remove sign-compare - sysflags="$($PYTHON -c "from distutils import sysconfig; \ + sysflags="$($PYTHON -c "import sysconfig; \ print (sysconfig.get_config_var('CFLAGS'))")" export CFLAGS="$sysflags $werrors -Wlogical-op -Wno-sign-compare" diff --git a/tools/wheels/check_license.py b/tools/wheels/check_license.py index 0fe7356c0..8ced317d6 100644 --- a/tools/wheels/check_license.py +++ b/tools/wheels/check_license.py @@ -9,7 +9,6 @@ distribution. """ import os import sys -import io import re import argparse @@ -36,7 +35,7 @@ def main(): # Check license text license_txt = os.path.join(os.path.dirname(mod.__file__), "LICENSE.txt") - with io.open(license_txt, "r", encoding="utf-8") as f: + with open(license_txt, encoding="utf-8") as f: text = f.read() ok = check_text(text) diff --git a/tools/wheels/cibw_before_build.sh b/tools/wheels/cibw_before_build.sh index ad76b330f..493cceeae 100644 --- a/tools/wheels/cibw_before_build.sh +++ b/tools/wheels/cibw_before_build.sh @@ -15,13 +15,18 @@ fi # Install Openblas if [[ $RUNNER_OS == "Linux" || $RUNNER_OS == "macOS" ]] ; then basedir=$(python tools/openblas_support.py) - cp -r $basedir/lib/* /usr/local/lib - cp $basedir/include/* /usr/local/include if [[ $RUNNER_OS == "macOS" && $PLATFORM == "macosx-arm64" ]]; then + # /usr/local/lib doesn't exist on cirrus-ci runners + sudo mkdir -p /usr/local/lib /usr/local/include /usr/local/lib/cmake/openblas sudo mkdir -p /opt/arm64-builds/lib /opt/arm64-builds/include sudo chown -R $USER /opt/arm64-builds cp -r $basedir/lib/* /opt/arm64-builds/lib cp $basedir/include/* /opt/arm64-builds/include + sudo cp -r $basedir/lib/* /usr/local/lib + sudo cp $basedir/include/* /usr/local/include + else + cp -r $basedir/lib/* /usr/local/lib + cp $basedir/include/* /usr/local/include fi elif [[ $RUNNER_OS == "Windows" ]]; then PYTHONPATH=tools python -c "import openblas_support; openblas_support.make_init('numpy')" @@ -42,6 +47,5 @@ if [[ $RUNNER_OS == "macOS" ]]; then fi source $PROJECT_DIR/tools/wheels/gfortran_utils.sh install_gfortran - # Try a newer version of delocate that knows about /usr/local/lib - pip install git+https://github.com/isuruf/delocate@search_usr_local#egg=delocate + pip install "delocate==0.10.4" fi diff --git a/tools/wheels/gfortran_utils.sh b/tools/wheels/gfortran_utils.sh index 39357b3fe..9102ba127 100644 --- a/tools/wheels/gfortran_utils.sh +++ b/tools/wheels/gfortran_utils.sh @@ -123,7 +123,7 @@ if [ "$(uname)" == "Darwin" ]; then curl -L -O https://github.com/isuruf/gcc/releases/download/gcc-11.3.0-2/gfortran-darwin-${arch}-${type}.tar.gz case ${arch}-${type} in arm64-native) - export GFORTRAN_SHA=142290685240f4f86cdf359cb2030d586105d7e4 + export GFORTRAN_SHA=0d5c118e5966d0fb9e7ddb49321f63cac1397ce8 ;; arm64-cross) export GFORTRAN_SHA=527232845abc5af21f21ceacc46fb19c190fe804 @@ -148,10 +148,10 @@ if [ "$(uname)" == "Darwin" ]; then if [[ "${type}" == "native" ]]; then # Link these into /usr/local so that there's no need to add rpath or -L for f in libgfortran.dylib libgfortran.5.dylib libgcc_s.1.dylib libgcc_s.1.1.dylib libquadmath.dylib libquadmath.0.dylib; do - ln -sf /opt/gfortran-darwin-${arch}-${type}/lib/$f /usr/local/lib/$f + sudo ln -sf /opt/gfortran-darwin-${arch}-${type}/lib/$f /usr/local/lib/$f done # Add it to PATH - ln -sf /opt/gfortran-darwin-${arch}-${type}/bin/gfortran /usr/local/bin/gfortran + sudo ln -sf /opt/gfortran-darwin-${arch}-${type}/bin/gfortran /usr/local/bin/gfortran fi } diff --git a/tools/wheels/upload_wheels.sh b/tools/wheels/upload_wheels.sh index 6694caf01..bc2066726 100644 --- a/tools/wheels/upload_wheels.sh +++ b/tools/wheels/upload_wheels.sh @@ -37,8 +37,6 @@ upload_wheels() { if [[ -z ${TOKEN} ]]; then echo no token set, not uploading else - python -m pip install \ - git+https://github.com/Anaconda-Platform/anaconda-client.git@be1e14936a8e947da94d026c990715f0596d7043 # sdists are located under dist folder when built through setup.py if compgen -G "./dist/*.gz"; then echo "Found sdist" |