diff options
| -rw-r--r-- | .github/workflows/docker.yml | 59 | ||||
| -rw-r--r-- | .github/workflows/gitpod.yml | 55 | ||||
| -rw-r--r-- | .gitpod.yml | 63 | ||||
| -rw-r--r-- | .hadolint.yaml | 7 | ||||
| -rw-r--r-- | environment.yml | 2 | ||||
| -rw-r--r-- | tools/gitpod/Dockerfile | 141 | ||||
| -rw-r--r-- | tools/gitpod/gitpod.Dockerfile | 45 | ||||
| -rw-r--r-- | tools/gitpod/settings.json | 9 | ||||
| -rw-r--r-- | tools/gitpod/workspace_config | 58 |
9 files changed, 376 insertions, 63 deletions
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..52d78a38d --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,59 @@ +name: Build Base Docker Image + +on: + push: + branches: + - main + paths: + - './environment.yml' + tags: + - '*' + +jobs: + build: + name: Build base Docker image + runs-on: ubuntu-latest + environment: numpy-dev + if: "github.repository_owner == 'numpy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" + steps: + - name: Clone repository + uses: actions/checkout@v2 + - name: Lint Docker + uses: brpaz/hadolint-action@v1.2.1 + with: + dockerfile: ./tools/gitpod/Dockerfile + - name: Get refs + shell: bash + run: | + export raw_branch=${GITHUB_REF#refs/heads/} + echo "::set-output name=branch::${raw_branch//\//-}" + echo "::set-output name=date::$(date +'%Y%m%d')" + echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)" + id: getrefs + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: ${{ runner.os }}-buildx- + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: "." + file: "./tools/gitpod/Dockerfile" + push: ${{ github.event_name != 'pull_request' }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + numpy/numpy-dev:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}, numpy/numpy-dev:latest + - name: Image digest + # Return details of the image build: sha and shell + run: echo ${{ steps.docker_build.outputs.digest }}
\ No newline at end of file diff --git a/.github/workflows/gitpod.yml b/.github/workflows/gitpod.yml new file mode 100644 index 000000000..55683bcae --- /dev/null +++ b/.github/workflows/gitpod.yml @@ -0,0 +1,55 @@ +name: Build Gitpod Docker image + +on: + push: + branches: + - main + +jobs: + build: + name: Build Gitpod Docker image + runs-on: ubuntu-latest + environment: numpy-dev + if: "github.repository_owner == 'numpy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')" + steps: + - name: Clone repository + uses: actions/checkout@v2 + - name: Lint Docker + uses: brpaz/hadolint-action@v1.2.1 + with: + dockerfile: ./tools/gitpod/gitpod.Dockerfile + - name: Get refs + shell: bash + run: | + export raw_branch=${GITHUB_REF#refs/heads/} + echo "::set-output name=branch::${raw_branch//\//-}" + echo "::set-output name=date::$(date +'%Y%m%d')" + echo "::set-output name=sha8::$(echo ${GITHUB_SHA} | cut -c1-8)" + id: getrefs + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: ${{ runner.os }}-buildx- + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + context: "." + file: "./tools/gitpod/gitpod.Dockerfile" + push: ${{ github.event_name != 'pull_request' }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + tags: | + numpy/numpy-gitpod:${{ steps.getrefs.outputs.date }}-${{ steps.getrefs.outputs.branch}}-${{ steps.getrefs.outputs.sha8 }}, numpy/numpy-gitpod:latest + - name: Image digest + # Return details of the image build: sha and shell + run: echo ${{ steps.docker_build.outputs.digest }}
\ No newline at end of file diff --git a/.gitpod.yml b/.gitpod.yml index c1755607b..dfbee831a 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,10 +1,63 @@ -image: thomasjpfan/numpy-gitpod:latest +# Rebuilding NumPy on init - rather than on prebuild: this ensures +# that even forks do have a usable freshly built NumPy +# Might delegate this later to prebuild with Q2 improvements on gitpod +# https://www.gitpod.io/docs/config-start-tasks/#configuring-the-terminal +# ------------------------------------------------------------------------- + +image: numpy/numpy-gitpod:latest tasks: - # The base image complied numpy with ccache enabled. This second build - # should be faster since it is using the cache. - - init: python setup.py build_ext -i + - name: Prepare development + init: | + mkdir -p .vscode + cp tools/gitpod/settings.json .vscode/settings.json + conda activate numpy-dev + python setup.py build_ext --inplace + echo "🛠Completed rebuilding NumPy!! 🛠" + echo "📖 Building docs 📖 " + git submodule update --init + cd doc + make html + echo "✨ Pre-build complete! You can close this terminal ✨ " + + +# -------------------------------------------------------- +# exposing ports for liveserve +ports: + - port: 5500 + onOpen: notify + +# -------------------------------------------------------- +# some useful extensions to have +vscode: + extensions: + - eamodio.gitlens + - njpwerner.autodocstring + - lextudio.restructuredtext + - ritwickdey.liveserver + - ms-python.python + - yzhang.markdown-all-in-one + - bungcip.better-toml + - mhutchie.git-graph +# -------------------------------------------------------- +# using prebuilds for the container - note: atm this only +# works for the NumPy repo +# With this configuration the prebuild will happen on push to master github: prebuilds: + # enable for main/default branch master: true - branches: true + # enable for other branches (defaults to false) + branches: false + # enable for pull requests coming from this repo (defaults to true) + pullRequests: false + # enable for pull requests coming from forks (defaults to false) + pullRequestsFromForks: false + # add a check to pull requests (defaults to true) + addCheck: false + # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) + addComment: false + # add a "Review in Gitpod" button to the pull request's description (defaults to false) + addBadge: false + # add a label once the prebuild is ready to pull requests (defaults to false) + addLabel: false
\ No newline at end of file diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 000000000..0188ba2cf --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,7 @@ +--- +ignored: + - DL3006 + - DL3008 + - SC2016 + - DL3004 + - DL3007
\ No newline at end of file diff --git a/environment.yml b/environment.yml index 3acc3dda5..19fe69a79 100644 --- a/environment.yml +++ b/environment.yml @@ -19,7 +19,7 @@ dependencies: - mypy=0.812 - typing_extensions # For building docs - - sphinx=3.5.2 + - sphinx=3.5.4 - numpydoc=1.1.0 - ipython - scipy diff --git a/tools/gitpod/Dockerfile b/tools/gitpod/Dockerfile index b9c0d4449..e2e0e1bc9 100644 --- a/tools/gitpod/Dockerfile +++ b/tools/gitpod/Dockerfile @@ -1,74 +1,101 @@ -# Builds a development environment for gitpod by building numpy with -# ccache enabled. When gitpod is prebuilding or starting up it clones -# a branch into `/workspace/numpy`. The gitpod clone will build numpy -# faster because it is using compliers with ccache enabled. -FROM gitpod/workspace-base as clone - -COPY --chown=gitpod . /tmp/numpy_repo - -# We use a multistage build to create a shallow clone of the repo to avoid -# having the complete git history in the build stage and reducing the image -# size. During the build stage, the shallow clone is used to install the -# dependencies and build numpy to populate the cache used by ccache. Building -# numpy with setup.py uses versioneer.py which requires a git history. -RUN git clone --depth 1 file:////tmp/numpy_repo /tmp/numpy - -FROM gitpod/workspace-base as build - -# gitpod/workspace-base needs at least one file here -RUN touch /home/gitpod/.bashrc.d/empty +# +# Dockerfile for NumPy development +# +# Usage: +# ------- +# +# To make a local build of the container, from the 'Docker-dev' directory: +# docker build --rm -f "Dockerfile" -t <build-tag> "." +# +# To use the container use the following command. It assumes that you are in +# the root folder of the NumPy git repository, making it available as +# /home/numpy in the container. Whatever changes you make to that directory +# are visible in the host and container. +# The docker image is retrieved from the NumPy dockerhub repository +# +# docker run --rm -it -v $(pwd):/home/numpy numpy/numpy-dev:<image-tag> +# +# By default the container will activate the conda environment numpy-dev +# which contains all the dependencies needed for NumPy development +# +# To build NumPy run: python setup.py build_ext --inplace +# +# To run the tests use: python runtests.py +# +# This image is based on: Ubuntu 20.04 (focal) +# https://hub.docker.com/_/ubuntu/?tab=tags&name=focal +# OS/ARCH: linux/amd64 +FROM gitpod/workspace-base:latest ARG MAMBAFORGE_VERSION="4.10.0-0" ARG CONDA_ENV=numpy-dev -ENV CONDA_DIR=/home/gitpod/mambaforge3 -ENV PATH=$CONDA_DIR/bin:$PATH +# ---- Configure environment ---- +ENV CONDA_DIR=/home/gitpod/mambaforge3 \ + SHELL=/bin/bash +ENV PATH=${CONDA_DIR}/bin:$PATH \ + WORKSPACE=/workspace/numpy + + +# ----------------------------------------------------------------------------- +# ---- Creating as root - note: make sure to change to gitpod in the end ---- USER root -RUN install-packages texlive-latex-extra dvisvgm -USER gitpod + +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get install -yq --no-install-recommends \ + ca-certificates \ + dirmngr \ + dvisvgm \ + gnupg \ + gpg-agent \ + texlive-latex-extra \ + vim && \ + # this needs to be done after installing dirmngr + apt-key adv --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 && \ + apt-add-repository https://cli.github.com/packages && \ + apt-get install -yq --no-install-recommends \ + gh && \ + locale-gen en_US.UTF-8 && \ + apt-get clean && \ + rm -rf /var/cache/apt/* &&\ + rm -rf /var/lib/apt/lists/* &&\ + rm -rf /tmp/* # Allows this Dockerfile to activate conda environments SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] -# Install mambaforge3 +# ----------------------------------------------------------------------------- +# ---- Installing mamba ---- RUN wget -q -O mambaforge3.sh \ - https://github.com/conda-forge/miniforge/releases/download/$MAMBAFORGE_VERSION/Mambaforge-$MAMBAFORGE_VERSION-Linux-x86_64.sh && \ - bash mambaforge3.sh -p $CONDA_DIR -b && \ + "https://github.com/conda-forge/miniforge/releases/download/$MAMBAFORGE_VERSION/Mambaforge-$MAMBAFORGE_VERSION-Linux-x86_64.sh" && \ + bash mambaforge3.sh -p ${CONDA_DIR} -b && \ rm mambaforge3.sh -# makes conda activate command for this Dockerfile -RUN echo ". $CONDA_DIR/etc/profile.d/conda.sh" >> ~/.profile -# enables conda for interactive sessions -RUN conda init bash +# ----------------------------------------------------------------------------- +# ---- Copy needed files ---- +# basic workspace configurations +COPY ./tools/gitpod/workspace_config /usr/local/bin/workspace_config -# Install numpy dev dependencies -COPY --from=clone --chown=gitpod /tmp/numpy /workspace/numpy -RUN mamba env create -f /workspace/numpy/environment.yml -n $CONDA_ENV && \ - conda activate $CONDA_ENV && \ - mamba install ccache -y && \ - conda clean --all -f -y +RUN chmod a+rx /usr/local/bin/workspace_config && \ + workspace_config -# Set up ccache for compilers for this Dockerfile and interactino sessions -# Using `conda env config vars set` does not work with Docker -# REF: https://github.com/conda-forge/compilers-feedstock/issues/31 -RUN echo "conda activate $CONDA_ENV" >> ~/.startuprc && \ - echo "export CC=\"ccache \$CC\"" >> ~/.startuprc && \ - echo "export CXX=\"ccache \$CXX\"" >> ~/.startuprc && \ - echo "export F77=\"ccache \$F77\"" >> ~/.startuprc && \ - echo "export F90=\"ccache \$F90\"" >> ~/.startuprc && \ - echo "export GFORTRAN=\"ccache \$GFORTRAN\"" >> ~/.startuprc && \ - echo "export FC=\"ccache \$FC\"" >> ~/.startuprc && \ - echo "source ~/.startuprc" >> ~/.profile && \ - echo "source ~/.startuprc" >> ~/.bashrc +# Copy conda environment file into the container - this needs to exists inside +# the container to create a conda environment from it +COPY environment.yml /tmp/environment.yml -# Build numpy to populate the cache used by ccache -RUN python /workspace/numpy/setup.py build_ext -i && \ - ccache -s - -# .gitpod.yml is configured to install numpy from /workspace/numpy -RUN echo "export PYTHONPATH=/workspace/numpy" >> ~/.bashrc +# ----------------------------------------------------------------------------- +# ---- Create conda environment ---- +# Install NumPy dependencies +RUN mamba env create -f /tmp/environment.yml && \ + conda activate ${CONDA_ENV} && \ + mamba install ccache -y && \ + # needed for docs rendering later on + python -m pip install --no-cache-dir sphinx-autobuild && \ + conda clean --all -f -y && \ + rm -rf /tmp/* -# gitpod will load the repository into /workspace/numpy. We remove the -# directoy from the image to prevent conflicts -RUN sudo rm -rf /workspace/numpy +# ----------------------------------------------------------------------------- +# Always make sure we are not root +USER gitpod
\ No newline at end of file diff --git a/tools/gitpod/gitpod.Dockerfile b/tools/gitpod/gitpod.Dockerfile new file mode 100644 index 000000000..ad731fd63 --- /dev/null +++ b/tools/gitpod/gitpod.Dockerfile @@ -0,0 +1,45 @@ +# Doing a local shallow clone - keeps the container secure +# and much slimmer than using COPY directly or making a +# remote clone +ARG BASE_CONTAINER="numpy/numpy-dev:latest" +FROM gitpod/workspace-base:latest as clone + +COPY --chown=gitpod . /tmp/numpy_repo +RUN git clone --depth 1 file:////tmp/numpy_repo /tmp/numpy + +# ----------------------------------------------------------------------------- +# Using the numpy-dev Docker image as a base +# This way, we ensure we have all the needed compilers and dependencies +# while reducing the build time +FROM ${BASE_CONTAINER} as build + +# ----------------------------------------------------------------------------- +USER root + +# ----------------------------------------------------------------------------- +# ---- ENV variables ---- +# ---- Directories needed ---- +ENV WORKSPACE=/workspace/numpy/ \ + CONDA_ENV=numpy-dev + +# Allows this Dockerfile to activate conda environments +SHELL ["/bin/bash", "--login", "-o", "pipefail", "-c"] + +# Copy over the shallow clone +COPY --from=clone --chown=gitpod /tmp/numpy ${WORKSPACE} + +# Everything happens in the /workspace/numpy directory +WORKDIR ${WORKSPACE} + +# Build numpy to populate the cache used by ccache +RUN conda activate ${CONDA_ENV} && \ + python setup.py build_ext --inplace && \ + ccache -s + +# Gitpod will load the repository into /workspace/numpy. We remove the +# directoy from the image to prevent conflicts +RUN rm -rf ${WORKSPACE} + +# ----------------------------------------------------------------------------- +# Always return to non privileged user +USER gitpod diff --git a/tools/gitpod/settings.json b/tools/gitpod/settings.json new file mode 100644 index 000000000..8f070c04c --- /dev/null +++ b/tools/gitpod/settings.json @@ -0,0 +1,9 @@ +{ + "restructuredtext.languageServer.disabled": true, + "restructuredtext.builtDocumentationPath": "${workspaceRoot}/doc/build/html", + "restructuredtext.confPath": "", + "restructuredtext.updateOnTextChanged": "true", + "restructuredtext.updateDelay": 300, + "restructuredtext.linter.disabled": true, + "python.pythonPath": "/home/gitpod/mambaforge3/envs/numpy-dev/bin/python" +}
\ No newline at end of file diff --git a/tools/gitpod/workspace_config b/tools/gitpod/workspace_config new file mode 100644 index 000000000..aa859c9be --- /dev/null +++ b/tools/gitpod/workspace_config @@ -0,0 +1,58 @@ +#!/bin/bash +# Basic configurations for the workspace + +set -e + +# gitpod/workspace-base needs at least one file here +touch /home/gitpod/.bashrc.d/empty + +# Add git aliases +git config --global alias.co checkout +git config --global alias.ci commit +git config --global alias.st status +git config --global alias.br branch +git config --global alias.hist "log --pretty=format:'%h %ad | %s%d [%an]' --graph --date=short" +git config --global alias.type 'cat-file -t' +git config --global alias.dump 'cat-file -p' + +# Enable basic vim defaults in ~/.vimrc +echo "filetype plugin indent on" >>~/.vimrc +echo "set colorcolumn=80" >>~/.vimrc +echo "set number" >>~/.vimrc +echo "syntax enable" >>~/.vimrc + +# Vanity custom bash prompt - makes it more legible +echo "PS1='\[\e]0;\u \w\a\]\[\033[01;36m\]\u\[\033[m\] > \[\033[38;5;141m\]\w\[\033[m\] \\$ '" >>~/.bashrc + +# Enable prompt color in the skeleton .bashrc +# hadolint ignore=SC2016 +sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc + +# .gitpod.yml is configured to install NumPy from /workspace/numpy +echo "export PYTHONPATH=${WORKSPACE}" >>~/.bashrc + +# make conda activate command available from /bin/bash (login and interactive) +if [[ ! -f "/etc/profile.d/conda.sh" ]]; then + ln -s ${CONDA_DIR}/etc/profile.d/conda.sh /etc/profile.d/conda.sh +fi +echo ". ${CONDA_DIR}/etc/profile.d/conda.sh" >>~/.bashrc +echo "conda activate numpy-dev" >>~/.bashrc + +# Enable prompt color in the skeleton .bashrc +# hadolint ignore=SC2016 +sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc + +# .gitpod.yml is configured to install numpy from /workspace/numpy +echo "export PYTHONPATH=/workspace/numpy" >>~/.bashrc + +# Set up ccache for compilers for this Dockerfile +# REF: https://github.com/conda-forge/compilers-feedstock/issues/31 +echo "conda activate numpy-dev" >>~/.startuprc +echo "export CC=\"ccache \$CC\"" >>~/.startuprc +echo "export CXX=\"ccache \$CXX\"" >>~/.startuprc +echo "export F77=\"ccache \$F77\"" >>~/.startuprc +echo "export F90=\"ccache \$F90\"" >>~/.startuprc +echo "export GFORTRAN=\"ccache \$GFORTRAN\"" >>~/.startuprc +echo "export FC=\"ccache \$FC\"" >>~/.startuprc +echo "source ~/.startuprc" >>~/.profile +echo "source ~/.startuprc" >>~/.bashrc |