From e8c711b0beb0599f0170ed11a32d31d0ca206790 Mon Sep 17 00:00:00 2001 From: moparisthebest Date: Thu, 26 Nov 2020 15:05:21 -0500 Subject: [PATCH] Re-factor self-ci scripts --- {.jenkins => .ci}/Jenkinsfile | 0 build-ci.sh | 76 ++++++++++++++++++++++++++++++----- c/build.sh | 27 +++++++++---- java/Dockerfile | 7 ++-- java/build.sh | 10 +++-- java/readme.md | 9 ++++- java/run-java | 2 +- java/run-java-all | 7 ++++ rust/build-jenkins.sh | 37 ----------------- rust/build.sh | 9 ++++- 10 files changed, 119 insertions(+), 65 deletions(-) rename {.jenkins => .ci}/Jenkinsfile (100%) create mode 100755 java/run-java-all delete mode 100755 rust/build-jenkins.sh diff --git a/.jenkins/Jenkinsfile b/.ci/Jenkinsfile similarity index 100% rename from .jenkins/Jenkinsfile rename to .ci/Jenkinsfile diff --git a/build-ci.sh b/build-ci.sh index d0719f1..ddf2b33 100755 --- a/build-ci.sh +++ b/build-ci.sh @@ -4,17 +4,69 @@ # 1. rust, this runs $BUILD_SCRIPT with env variable TARGET and DISABLE_TESTS set appropriately once for each target supported by cross # this'll have to do until cross can run in docker containers backed by the btrfs driver... # 2. c, this runs $BUILD_SCRIPT inside an alpine linux container, once for each ARCH supported by build.sh here +# 3. java, this runs $BUILD_SCRIPT inside docker container moparisthebest/self-ci-java:latest once for each version of Java currently supported # run like (for rust): -# curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/build-ci.sh | bash -s -- rust -# curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/build-ci.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- rust +# curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/build-ci.sh | bash -s -- -l rust +# curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/build-ci.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- -l rust -export BUILD_LANG="${1-rust}" -shift -export BUILD_SCRIPT="${1-.jenkins/build.sh}" -shift -export RELEASE_SCRIPT="${1-.jenkins/release.sh}" -shift +BUILD_LANG='' +RELEASE_SCRIPT='.ci/release.sh' +BUILD_SCRIPT='.ci/build.sh' +DEFAULT_RELEASE_DIR='release' + +# compat with alternative directory name +[ ! -f "$RELEASE_SCRIPT" ] && [ -f '.jenkins/release.sh' ] && RELEASE_SCRIPT='.jenkins/release.sh' +[ ! -f "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh' + +function usage { + echo "$@" + echo + echo "Usage: build-ci.sh [-l BUILD_LANG] [-r RELEASE_SCRIPT] [-d DEFAULT_RELEASE_DIR] [-b BUILD_SCRIPT]" 2>&1 + echo 'Build a project with self-ci.' + echo ' -l BUILD_LANG Specify the language build script to grab+execute (must be one of: rust, c, java)' + echo ' -r RELEASE_SCRIPT Specify the release script in this directory to run after build is successful' + echo ' -d DEFAULT_RELEASE_DIR If RELEASE_SCRIPT not specified, and this directory exists after successful build' + echo ' then run default release script on all files in it' + echo ' -b BUILD_SCRIPT Specify the build script in this directory to run with above language script' + exit 1 +} + +while getopts 'l:b:r:d:' arg; do + case ${arg} in + l) + BUILD_LANG="${OPTARG}" + [ "" == "$(echo "$BUILD_LANG" | grep -E '^(rust|c|java)$')" ] && usage "Invalid lang $BUILD_LANG, must be one of: rust, c, java" + ;; + b) + BUILD_SCRIPT="${OPTARG}" + ;; + r) + RELEASE_SCRIPT="${OPTARG}" + [ ! -e "$RELEASE_SCRIPT" ] && usage "Release script $RELEASE_SCRIPT does not exist" + ;; + d) + DEFAULT_RELEASE_DIR="${OPTARG}" + ;; + ?) + usage "Invalid option: -${OPTARG}." + ;; + esac +done + +shift "$((OPTIND-1))" + +# for all currently supported langs, this must exist in *this* directory +[ ! -e "$BUILD_SCRIPT" ] && usage "Build script $BUILD_SCRIPT does not exist" + +# a bit of deterministic smarts to auto-detect BUILD_LANG +[ "$BUILD_LANG" == "" ] && [ -e pom.xml -o -e build.gradle ] && BUILD_LANG=java +[ "$BUILD_LANG" == "" ] && [ -e Cargo.toml ] && BUILD_LANG=rust +[ "$BUILD_LANG" == "" ] && [ -e Makefile ] && BUILD_LANG=c + +[ "$BUILD_LANG" == "" ] && usage "-l BUILD_LANG autodetection failed, must be set manually" + +echo "build-ci.sh: BUILD_LANG=$BUILD_LANG RELEASE_SCRIPT=$RELEASE_SCRIPT BUILD_SCRIPT=$BUILD_SCRIPT remaining args:" "$@" set -euxo pipefail @@ -23,18 +75,22 @@ cd bin curl --compressed -sL -O 'https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/ci-release-helper.sh' -O "https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/$BUILD_LANG/build.sh" chmod +x ci-release-helper.sh build.sh cd .. + export PATH="$(pwd)/bin:$PATH" +export BRANCH_NAME="$BRANCH_NAME" +export BUILD_UID="$UID" +export BUILD_GID="$(id -g)" build.sh "$BUILD_SCRIPT" "$@" if [ -e "$RELEASE_SCRIPT" ] then "$RELEASE_SCRIPT" "$@" -elif [ -d 'release' ] +elif [ -d "$DEFAULT_RELEASE_DIR" ] then # default release script ci-release-helper.sh standard_pre_release - cd release + cd "$DEFAULT_RELEASE_DIR" find -type f ! -path ./sha256sum.txt -print0 | xargs -0 sha256sum > sha256sum.txt gpg --clearsign sha256sum.txt ci-release-helper.sh standard_multi_release 'sha256sum.txt' 'text/plain' diff --git a/c/build.sh b/c/build.sh index e7aec01..8ebeddc 100755 --- a/c/build.sh +++ b/c/build.sh @@ -1,23 +1,36 @@ #!/bin/sh set -euxo pipefail -export BUILD_SCRIPT="${1-.jenkins/build.sh}" +BUILD_SCRIPT="$1" +shift +[ "" == "$BUILD_SCRIPT" ] && [ -f '.ci/build.sh' ] && BUILD_SCRIPT='.ci/build.sh' +[ "" == "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh' + + +# this assists in cleanup, ie if a job was terminated, to be run in a jenkins finally {} or similar +if [ "$BUILD_SCRIPT" == "docker-chown" ] +then + # chown everything in this directory to this uid/gid + docker run --rm -v "$(pwd)":/tmp alpine chown -R "$UID:$(id -g)" /tmp + exit $? +fi docker_build() { export ARCH="$1" shift DOCKER_IMAGE="$1" + shift # run it, but after, chown anything left in /tmp to *this* uid/gid, otherwise we can't delete them later... - docker run --rm -e ARCH -v "$(pwd)":/tmp "$DOCKER_IMAGE" sh -c "'/tmp/$BUILD_SCRIPT'; exit=\$?; chown -R '$UID:$(id -g)' /tmp; exit \$exit" + docker run --rm -e ARCH -e BRANCH_NAME -e BUILD_UID="$UID" -e BUILD_GID="$(id -g)" -v "$(pwd)":/tmp -w /tmp "$DOCKER_IMAGE" sh -c "\"\$@\"; exit=\$?; chown -R '$UID:$(id -g)' /tmp; exit \$exit" -- "$@" } -docker_build 'amd64' 'alpine' +docker_build 'amd64' 'alpine' "$BUILD_SCRIPT" "$@" # before first multiarch image, must register binfmt handlers docker run --rm --privileged multiarch/qemu-user-static:register --reset -docker_build 'i386' 'multiarch/alpine:i386-latest-stable' -docker_build 'aarch64' 'multiarch/alpine:aarch64-latest-stable' -docker_build 'armv7' 'multiarch/alpine:armv7-latest-stable' -docker_build 'ppc64le' 'multiarch/alpine:ppc64le-latest-stable' +docker_build 'i386' 'multiarch/alpine:i386-latest-stable' "$BUILD_SCRIPT" "$@" +docker_build 'aarch64' 'multiarch/alpine:aarch64-latest-stable' "$BUILD_SCRIPT" "$@" +docker_build 'armv7' 'multiarch/alpine:armv7-latest-stable' "$BUILD_SCRIPT" "$@" +docker_build 'ppc64le' 'multiarch/alpine:ppc64le-latest-stable' "$BUILD_SCRIPT" "$@" diff --git a/java/Dockerfile b/java/Dockerfile index c34fee0..7788f91 100644 --- a/java/Dockerfile +++ b/java/Dockerfile @@ -22,10 +22,11 @@ RUN mkdir /m2 /npm && ln -sf /m2/ /root/.m2 && ln -sf /npm/ /root/.npm && \ curl https://download.java.net/java/early_access/jdk16/23/GPL/openjdk-16-ea+23_linux-x64_bin.tar.gz | bsdtar -xf - -C /usr/lib/jvm && \ mv /usr/lib/jvm/java-se-9-ri/jdk-9 /usr/lib/jvm/ && rm -rf java-se-9-ri && chmod +x /usr/lib/jvm/jdk-9/bin/* -COPY ./build.sh ./run-java /usr/bin/ +COPY ./run-java-all ./run-java /usr/bin/ +RUN ln -sf /usr/bin/run-java-all /usr/bin/build.sh -VOLUME [ "/build", "/m2", "/npm" ] +VOLUME [ "/build", "/m2", "/npm", "/root/.netrc" ] WORKDIR /build ENTRYPOINT ["/usr/bin/run.sh"] -CMD ["/usr/bin/build.sh", "./.jenkins/build.sh"] +CMD ["/usr/bin/run-java-all", "./.jenkins/build.sh"] diff --git a/java/build.sh b/java/build.sh index e332db5..75f2c1e 100755 --- a/java/build.sh +++ b/java/build.sh @@ -1,7 +1,9 @@ #!/bin/bash set -euo pipefail -for jdk in ${BUILD_JDKS:-6 7 8 9 10 11 12 13 14 15 16} -do - run-java "$jdk" "$@" -done +BUILD_SCRIPT="$1" +shift +[ "" == "$BUILD_SCRIPT" ] && [ -f '.ci/build.sh' ] && BUILD_SCRIPT='.ci/build.sh' +[ "" == "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh' + +docker run --rm -v "$HOME/.netrc:/root/.netrc:ro" -v "$HOME/.m2:/m2" -v "$HOME/.npm:/npm" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest /usr/bin/run-java-all "$BUILD_SCRIPT" "$@" diff --git a/java/readme.md b/java/readme.md index 84d4586..37f54d9 100644 --- a/java/readme.md +++ b/java/readme.md @@ -6,11 +6,18 @@ A docker container with every version of the Java JDK 6 to 16 installed in it, a Meant to be ran in CI something like this: ```sh +# docker directly docker run --rm -v "$HOME/.m2:/m2" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest + +# through self-ci script hosted at code.moparisthebest.com +curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/build-ci.sh | bash -s -- java + +# through self-ci script hosted at github +curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/build-ci.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- java ``` Without arguments it will execute `.jenkins/build.sh` once for each version of Java installed, setting the env variables JAVA_VERSION (a number), JAVA_HOME, M2_HOME, and PATH appropriately so invocations of `mvn` and `java` *just work*. If you want to call another script each time: ```sh -docker run --rm -v "$HOME/.m2:/m2" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest build.sh ./path/to/your/script.sh +docker run --rm -v "$HOME/.m2:/m2" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest run-java-all ./path/to/your/script.sh ``` diff --git a/java/run-java b/java/run-java index 1594e5b..7c4d962 100755 --- a/java/run-java +++ b/java/run-java @@ -60,6 +60,6 @@ else fi # maven 3.2.5 is the latest version supported by 6 -[ $JAVA_VERSION -eq 6 ] && export M2_HOME="/opt/apache-maven-3.2.5" +[ $JAVA_VERSION -eq 6 ] && M2_HOME="/opt/apache-maven-3.2.5" JAVA_VERSION=$JAVA_VERSION M2_HOME="$M2_HOME" JAVA_HOME="$JAVA_HOME" PATH="$JAVA_HOME/bin:$M2_HOME/bin:$PATH" exec "$@" diff --git a/java/run-java-all b/java/run-java-all new file mode 100755 index 0000000..e332db5 --- /dev/null +++ b/java/run-java-all @@ -0,0 +1,7 @@ +#!/bin/bash +set -euo pipefail + +for jdk in ${BUILD_JDKS:-6 7 8 9 10 11 12 13 14 15 16} +do + run-java "$jdk" "$@" +done diff --git a/rust/build-jenkins.sh b/rust/build-jenkins.sh deleted file mode 100755 index ec7106b..0000000 --- a/rust/build-jenkins.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -# this runs first arg or .jenkins/build.sh with env variable TARGET and DISABLE_TESTS set appropriately once for each target supported by cross -# this'll have to do until cross can run in docker containers backed by the btrfs driver... - -# run like: -# curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/rust/build-jenkins.sh | bash -# curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/rust/build-jenkins.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -# or with custom script: -# curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/rust/build-jenkins.sh | bash -s -- ./path/to/build.sh -# curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/rust/build-jenkins.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- ./path/to/build.sh - -export BUILD_SCRIPT="${1-.jenkins/build.sh}" -shift -export RELEASE_SCRIPT="${1-.jenkins/release.sh}" - -mkdir bin -cd bin -curl --compressed -sL -O https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/ci-release-helper.sh -O https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/rust/build.sh -chmod +x ci-release-helper.sh build.sh -cd .. -export PATH="$(pwd)/bin:$PATH" - -build.sh "$BUILD_SCRIPT" - -if [ -e "$RELEASE_SCRIPT" ] -then - "$RELEASE_SCRIPT" -elif [ -d 'release' ] -then - # default release script - ci-release-helper.sh standard_pre_release - find release/ -type f -print0 | xargs -0n1 -I {} ci-release-helper.sh standard_multi_release '{}' 'application/octet-stream' -fi - -exit 0 diff --git a/rust/build.sh b/rust/build.sh index ca16a6b..6e1deb9 100755 --- a/rust/build.sh +++ b/rust/build.sh @@ -1,14 +1,19 @@ #!/bin/bash set -euo pipefail +BUILD_SCRIPT="$1" +shift +[ "" == "$BUILD_SCRIPT" ] && [ -f '.ci/build.sh' ] && BUILD_SCRIPT='.ci/build.sh' +[ "" == "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh' + export CROSS_VERSION=0.2.1 for TARGET in ${BUILD_TARGETS:-x86_64-unknown-linux-musl x86_64-unknown-linux-gnu i686-unknown-linux-musl i686-unknown-linux-gnu i586-unknown-linux-musl i586-unknown-linux-gnu aarch64-unknown-linux-musl aarch64-unknown-linux-gnu armv7-unknown-linux-gnueabihf armv7-unknown-linux-musleabihf arm-unknown-linux-gnueabi arm-unknown-linux-gnueabihf arm-unknown-linux-musleabi arm-unknown-linux-musleabihf armv5te-unknown-linux-gnueabi armv5te-unknown-linux-musleabi x86_64-pc-windows-gnu x86_64-linux-android i686-linux-android aarch64-linux-android armv7-linux-androideabi arm-linux-androideabi mips64el-unknown-linux-gnuabi64 mips64-unknown-linux-gnuabi64 mipsel-unknown-linux-gnu mipsel-unknown-linux-musl mips-unknown-linux-gnu mips-unknown-linux-musl powerpc64le-unknown-linux-gnu powerpc-unknown-linux-gnu riscv64gc-unknown-linux-gnu s390x-unknown-linux-gnu x86_64-sun-solaris sparcv9-sun-solaris x86_64-unknown-netbsd} do if echo "$TARGET" | grep -E '(^s390x|^thumb|solaris$|^x86_64-unknown-dragonfly$|^x86_64-unknown-netbsd$)' >/dev/null then - DISABLE_TESTS=1 TARGET="$TARGET" "$@" + DISABLE_TESTS=1 TARGET="$TARGET" "$BUILD_SCRIPT" "$@" else - TARGET="$TARGET" "$@" + TARGET="$TARGET" "$BUILD_SCRIPT" "$@" fi done