diff options
Diffstat (limited to 'tools/ci')
| -rw-r--r-- | tools/ci/cirrus_general.yml | 99 | ||||
| -rw-r--r-- | tools/ci/cirrus_macosx_arm64.yml | 55 | ||||
| -rw-r--r-- | tools/ci/cirrus_wheels.yml | 157 | ||||
| -rwxr-xr-x | tools/ci/push_docs_to_repo.py | 7 |
4 files changed, 219 insertions, 99 deletions
diff --git a/tools/ci/cirrus_general.yml b/tools/ci/cirrus_general.yml deleted file mode 100644 index 2ec2ad837..000000000 --- a/tools/ci/cirrus_general.yml +++ /dev/null @@ -1,99 +0,0 @@ -build_and_store_wheels: &BUILD_AND_STORE_WHEELS - install_cibuildwheel_script: - - python -m pip install cibuildwheel==2.11.2 - cibuildwheel_script: - - cibuildwheel - wheels_artifacts: - path: "wheelhouse/*" - - -###################################################################### -# Build linux_aarch64 natively -###################################################################### - -linux_aarch64_task: - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder-arm64 - architecture: arm64 - platform: linux - cpu: 2 - memory: 8G - matrix: - # build in a matrix because building and testing all four wheels in a - # single task takes longer than 60 mins (the default time limit for a - # cirrus-ci task). - - env: - CIBW_BUILD: cp38-* - EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM - - env: - CIBW_BUILD: cp39-* - - env: - CIBW_BUILD: cp310-* - - env: - CIBW_BUILD: cp311-* - - build_script: | - apt install -y python3-venv python-is-python3 gfortran libatlas-base-dev libgfortran5 eatmydata - git fetch origin - ./tools/travis-before-install.sh - which python - echo $CIRRUS_CHANGE_MESSAGE - <<: *BUILD_AND_STORE_WHEELS - - -###################################################################### -# Upload all wheels -###################################################################### - -wheels_upload_task: - # Artifacts don't seem to be persistent from task to task. - # Rather than upload wheels at the end of each cibuildwheel run we do a - # final upload here. This is because a run may be on different OS for - # which bash, etc, may not be present. - depends_on: - - linux_aarch64 - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder - platform: linux - cpu: 1 - - env: - NUMPY_STAGING_UPLOAD_TOKEN: ENCRYPTED[!5a69522ae0c2af9edb2bc1cdfeaca6292fb3666d9ecd82dca0615921834a6ce3b702352835d8bde4ea2a9ed5ef8424ac!] - NUMPY_NIGHTLY_UPLOAD_TOKEN: ENCRYPTED[!196422e6c3419a3b1d79815e1026094a215cb0f346fe34ed0f9d3ca1c19339df7398d04556491b1e0420fc1fe3713289!] - - upload_script: | - apt-get install -y python3-venv python-is-python3 curl - export IS_SCHEDULE_DISPATCH="false" - export IS_PUSH="false" - - # cron job - if [[ "$CIRRUS_CRON" == "weekly" ]]; then - export IS_SCHEDULE_DISPATCH="true" - fi - - # a manual build was started - if [[ "$CIRRUS_BUILD_SOURCE" == "api" && "$CIRRUS_COMMIT_MESSAGE" == "API build for null" ]]; then - export IS_SCHEDULE_DISPATCH="true" - fi - - # only upload wheels to staging if it's a tag beginning with 'v' and you're - # on a maintenance branch - if [[ "$CIRRUS_TAG" == v* ]] && [[ "$CIRRUS_BRANCH" == maintenance* ]]; then - export IS_PUSH="true" - fi - - # The name of the zip file is derived from the `wheels_artifact` line. - # If you change the artifact line to `myfile_artifact` then it would be - # called myfile.zip - - curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/wheels.zip --output wheels.zip - unzip wheels.zip - - source ./tools/wheels/upload_wheels.sh - # IS_PUSH takes precedence over IS_SCHEDULE_DISPATCH - set_upload_vars - - # Will be skipped if not a push/tag/scheduled build - upload_wheels diff --git a/tools/ci/cirrus_macosx_arm64.yml b/tools/ci/cirrus_macosx_arm64.yml new file mode 100644 index 000000000..2343b807f --- /dev/null +++ b/tools/ci/cirrus_macosx_arm64.yml @@ -0,0 +1,55 @@ +modified_clone: &MODIFIED_CLONE + # makes sure that for a PR the CI runs against a merged main + clone_script: | + if [ -z "$CIRRUS_PR" ]; then + # if you're not in a PR then clone against the branch name that was pushed to. + git clone --recursive --branch=$CIRRUS_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR + git reset --hard $CIRRUS_CHANGE_IN_REPO + else + # it's a PR so clone the main branch then merge the changes from the PR + git clone https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR + git fetch origin pull/$CIRRUS_PR/head:pull/$CIRRUS_PR + + # CIRRUS_BASE_BRANCH will probably be `main` for the majority of the time + # However, if you do a PR against a maintenance branch we will want to + # merge the PR into the maintenance branch, not main + git checkout $CIRRUS_BASE_BRANCH + + # alpine git package needs default user.name and user.email to be set before a merge + git -c user.email="you@example.com" merge --no-commit pull/$CIRRUS_PR + git submodule update --init --recursive + fi + + +macos_arm64_test_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:14 + + <<: *MODIFIED_CLONE + + pip_cache: + folder: ~/.cache/pip + + test_script: | + brew install python@3.10 + + export PATH=/opt/homebrew/opt/python@3.10/libexec/bin:$PATH + python --version + + RUNNER_OS="macOS" + CFLAGS="-std=c99 -fno-strict-aliasing" + SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk + + # used for installing OpenBLAS/gfortran + bash tools/wheels/cibw_before_build.sh $PWD + + pushd ~/ + python -m venv numpy-dev + source numpy-dev/bin/activate + popd + + pip install -r build_requirements.txt + pip install pytest hypothesis typing_extensions + + spin build + spin test diff --git a/tools/ci/cirrus_wheels.yml b/tools/ci/cirrus_wheels.yml new file mode 100644 index 000000000..60512afab --- /dev/null +++ b/tools/ci/cirrus_wheels.yml @@ -0,0 +1,157 @@ +build_and_store_wheels: &BUILD_AND_STORE_WHEELS + install_cibuildwheel_script: + - python -m pip install cibuildwheel==2.12.1 + cibuildwheel_script: + - cibuildwheel + wheels_artifacts: + path: "wheelhouse/*" + +###################################################################### +# Build linux_aarch64 natively +###################################################################### + +linux_aarch64_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder-arm64 + architecture: arm64 + platform: linux + cpu: 2 + memory: 8G + matrix: + # build in a matrix because building and testing all four wheels in a + # single task takes longer than 60 mins (the default time limit for a + # cirrus-ci task). + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp39-* + EXPECT_CPU_FEATURES: NEON NEON_FP16 NEON_VFPV4 ASIMD ASIMDHP ASIMDDP ASIMDFHM + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp310-* + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp311-* + + build_script: | + apt install -y python3-venv python-is-python3 gfortran libatlas-base-dev libgfortran5 eatmydata + git fetch origin + ./tools/travis-before-install.sh + which python + echo $CIRRUS_CHANGE_MESSAGE + <<: *BUILD_AND_STORE_WHEELS + + +###################################################################### +# Build macosx_arm64 natively +###################################################################### + +macosx_arm64_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-monterey-xcode:14 + matrix: + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp39-* + - env: + CIRRUS_CLONE_SUBMODULES: true + CIBW_BUILD: cp310-* cp311-* + env: + PATH: /opt/homebrew/opt/python@3.10/bin:/usr/local/lib:/usr/local/include:$PATH + CIBW_ARCHS: arm64 + # Specifying CIBW_ENVIRONMENT_MACOS overrides pyproject.toml, so include + # all the settings from there, otherwise they're lost. + # SDKROOT needs to be set for repackaged conda-forge gfortran compilers + # supplied by isuruf. + # Find out SDKROOT via `xcrun --sdk macosx --show-sdk-path` + CIBW_ENVIRONMENT_MACOS: > + RUNNER_OS=macOS + SDKROOT=/Applications/Xcode-14.0.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX12.3.sdk + LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + CFLAGS="-std=c99 -fno-strict-aliasing" + OPENBLAS64_="/usr/local" + NPY_USE_BLAS_ILP64="1" + + build_script: + - brew install python@3.10 + - ln -s python3 /opt/homebrew/opt/python@3.10/bin/python + - which python + # needed for submodules + - git submodule update --init + # need to obtain all the tags so setup.py can determine FULLVERSION + - git fetch origin + - uname -m + - python -c "import platform;print(platform.python_version());print(platform.system());print(platform.machine())" + - clang --version + <<: *BUILD_AND_STORE_WHEELS + + +###################################################################### +# Upload all wheels +###################################################################### + +wheels_upload_task: + # Artifacts don't seem to be persistent from task to task. + # Rather than upload wheels at the end of each cibuildwheel run we do a + # final upload here. This is because a run may be on different OS for + # which bash, etc, may not be present. + depends_on: + - linux_aarch64 + - macosx_arm64 + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + + env: + NUMPY_STAGING_UPLOAD_TOKEN: ENCRYPTED[!5a69522ae0c2af9edb2bc1cdfeaca6292fb3666d9ecd82dca0615921834a6ce3b702352835d8bde4ea2a9ed5ef8424ac!] + NUMPY_NIGHTLY_UPLOAD_TOKEN: ENCRYPTED[!196422e6c3419a3b1d79815e1026094a215cb0f346fe34ed0f9d3ca1c19339df7398d04556491b1e0420fc1fe3713289!] + + upload_script: | + apt-get update + apt-get install -y curl wget + export IS_SCHEDULE_DISPATCH="false" + export IS_PUSH="false" + + # cron job + if [[ "$CIRRUS_CRON" == "weekly" ]]; then + export IS_SCHEDULE_DISPATCH="true" + fi + + # a manual build was started + if [[ "$CIRRUS_BUILD_SOURCE" == "api" && "$CIRRUS_COMMIT_MESSAGE" == "API build for null" ]]; then + export IS_SCHEDULE_DISPATCH="true" + fi + + # only upload wheels to staging if it's a tag beginning with 'v' and you're + # on a maintenance branch + if [[ "$CIRRUS_TAG" == v* ]] && [[ $CIRRUS_TAG != *"dev0"* ]]; then + export IS_PUSH="true" + fi + + if [[ $IS_PUSH == "true" ]] || [[ $IS_SCHEDULE_DISPATCH == "true" ]]; then + # install miniconda in the home directory. For some reason HOME isn't set by Cirrus + export HOME=$PWD + + # install miniconda for uploading to anaconda + wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda3 + $HOME/miniconda3/bin/conda init bash + source $HOME/miniconda3/bin/activate + conda install -y anaconda-client + + # The name of the zip file is derived from the `wheels_artifact` line. + # If you change the artifact line to `myfile_artifact` then it would be + # called myfile.zip + + curl https://api.cirrus-ci.com/v1/artifact/build/$CIRRUS_BUILD_ID/wheels.zip --output wheels.zip + unzip wheels.zip + + source ./tools/wheels/upload_wheels.sh + # IS_PUSH takes precedence over IS_SCHEDULE_DISPATCH + set_upload_vars + + # Will be skipped if not a push/tag/scheduled build + upload_wheels + fi diff --git a/tools/ci/push_docs_to_repo.py b/tools/ci/push_docs_to_repo.py index e34522a29..0471e3824 100755 --- a/tools/ci/push_docs_to_repo.py +++ b/tools/ci/push_docs_to_repo.py @@ -19,6 +19,8 @@ parser.add_argument('--committer', default='numpy-commit-bot', help='Name of the git committer') parser.add_argument('--email', default='numpy-commit-bot@nomail', help='Email of the git committer') +parser.add_argument('--count', default=1, type=int, + help="minimum number of expected files, defaults to 1") parser.add_argument( '--force', action='store_true', @@ -31,6 +33,11 @@ if not os.path.exists(args.dir): print('Content directory does not exist') sys.exit(1) +count = len([name for name in os.listdir(args.dir) if os.path.isfile(os.path.join(args.dir, name))]) + +if count < args.count: + print(f"Expected {args.count} top-directory files to upload, got {count}") + sys.exit(1) def run(cmd, stdout=True): pipe = None if stdout else subprocess.DEVNULL |