Re-factor self-ci scripts

2020-11-26 15:05:21 -05:00 · 2020-11-26 15:05:21 -05:00 · e8c711b0be
parent 7f3caffe7a
commit e8c711b0be
10 changed files with 119 additions and 65 deletions
--- a/.jenkins/Jenkinsfile
+++ b/.jenkins/Jenkinsfile
--- a/build-ci.sh
+++ b/build-ci.sh
@ -4,17 +4,69 @@
 # 1. rust, this runs $BUILD_SCRIPT with env variable TARGET and DISABLE_TESTS set appropriately once for each target supported by cross
 #    this'll have to do until cross can run in docker containers backed by the btrfs driver...
 # 2. c, this runs $BUILD_SCRIPT inside an alpine linux container, once for each ARCH supported by build.sh here
+# 3. java, this runs $BUILD_SCRIPT inside docker container moparisthebest/self-ci-java:latest once for each version of Java currently supported

 # run like (for rust):
-#   curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/build-ci.sh | bash -s -- rust
-#   curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/build-ci.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- rust
+#   curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/build-ci.sh | bash -s -- -l rust
+#   curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/build-ci.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- -l rust

-export BUILD_LANG="${1-rust}"
-shift
-export BUILD_SCRIPT="${1-.jenkins/build.sh}"
-shift
-export RELEASE_SCRIPT="${1-.jenkins/release.sh}"
-shift
+BUILD_LANG=''
+RELEASE_SCRIPT='.ci/release.sh'
+BUILD_SCRIPT='.ci/build.sh'
+DEFAULT_RELEASE_DIR='release'
+
+# compat with alternative directory name
+[ ! -f "$RELEASE_SCRIPT" ] && [ -f '.jenkins/release.sh' ] && RELEASE_SCRIPT='.jenkins/release.sh'
+[ ! -f "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh'
+
+function usage {
+        echo "$@"
+        echo
+        echo "Usage: build-ci.sh [-l BUILD_LANG] [-r RELEASE_SCRIPT] [-d DEFAULT_RELEASE_DIR] [-b BUILD_SCRIPT]" 2>&1
+        echo 'Build a project with self-ci.'
+        echo '   -l BUILD_LANG           Specify the language build script to grab+execute (must be one of: rust, c, java)'
+        echo '   -r RELEASE_SCRIPT       Specify the release script in this directory to run after build is successful'
+        echo '   -d DEFAULT_RELEASE_DIR  If RELEASE_SCRIPT not specified, and this directory exists after successful build'
+        echo '                           then run default release script on all files in it'
+        echo '   -b BUILD_SCRIPT         Specify the build script in this directory to run with above language script'
+        exit 1
+}
+
+while getopts 'l:b:r:d:' arg; do
+  case ${arg} in
+    l)
+      BUILD_LANG="${OPTARG}"
+      [ "" == "$(echo "$BUILD_LANG" | grep -E '^(rust|c|java)$')" ] && usage "Invalid lang $BUILD_LANG, must be one of: rust, c, java"
+      ;;
+    b)
+      BUILD_SCRIPT="${OPTARG}"
+      ;;
+    r)
+      RELEASE_SCRIPT="${OPTARG}"
+      [ ! -e "$RELEASE_SCRIPT" ] && usage "Release script $RELEASE_SCRIPT does not exist"
+      ;;
+    d)
+      DEFAULT_RELEASE_DIR="${OPTARG}"
+      ;;
+    ?)
+      usage "Invalid option: -${OPTARG}."
+      ;;
+  esac
+done
+
+shift "$((OPTIND-1))"
+
+# for all currently supported langs, this must exist in *this* directory
+[ ! -e "$BUILD_SCRIPT" ] && usage "Build script $BUILD_SCRIPT does not exist"
+
+# a bit of deterministic smarts to auto-detect BUILD_LANG
+[ "$BUILD_LANG" == "" ] && [ -e pom.xml -o -e build.gradle ] && BUILD_LANG=java
+[ "$BUILD_LANG" == "" ] && [ -e Cargo.toml ] && BUILD_LANG=rust
+[ "$BUILD_LANG" == "" ] && [ -e Makefile ] && BUILD_LANG=c
+
+[ "$BUILD_LANG" == "" ] && usage "-l BUILD_LANG autodetection failed, must be set manually"
+
+echo "build-ci.sh: BUILD_LANG=$BUILD_LANG RELEASE_SCRIPT=$RELEASE_SCRIPT BUILD_SCRIPT=$BUILD_SCRIPT remaining args:" "$@"

 set -euxo pipefail

@ -23,18 +75,22 @@ cd bin
 curl --compressed -sL -O 'https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/ci-release-helper.sh' -O "https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/$BUILD_LANG/build.sh"
 chmod +x ci-release-helper.sh build.sh
 cd ..
+
 export PATH="$(pwd)/bin:$PATH"
+export BRANCH_NAME="$BRANCH_NAME"
+export BUILD_UID="$UID"
+export BUILD_GID="$(id -g)"

 build.sh "$BUILD_SCRIPT" "$@"

 if [ -e "$RELEASE_SCRIPT" ]
 then
    "$RELEASE_SCRIPT" "$@"
-elif [ -d 'release' ]
+elif [ -d "$DEFAULT_RELEASE_DIR" ]
 then
    # default release script
    ci-release-helper.sh standard_pre_release
-    cd release
+    cd "$DEFAULT_RELEASE_DIR"
    find -type f ! -path ./sha256sum.txt -print0 | xargs -0 sha256sum > sha256sum.txt
    gpg --clearsign sha256sum.txt
    ci-release-helper.sh standard_multi_release 'sha256sum.txt' 'text/plain'
--- a/c/build.sh
+++ b/c/build.sh
@ -1,23 +1,36 @@
 #!/bin/sh
 set -euxo pipefail

-export BUILD_SCRIPT="${1-.jenkins/build.sh}"
+BUILD_SCRIPT="$1"
+shift
+[ "" == "$BUILD_SCRIPT" ] && [ -f '.ci/build.sh' ] && BUILD_SCRIPT='.ci/build.sh'
+[ "" == "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh'
+
+
+# this assists in cleanup, ie if a job was terminated, to be run in a jenkins finally {} or similar
+if [ "$BUILD_SCRIPT" == "docker-chown" ]
+then
+    # chown everything in this directory to this uid/gid
+    docker run --rm -v "$(pwd)":/tmp alpine chown -R "$UID:$(id -g)" /tmp
+    exit $?
+fi

 docker_build() {
    export ARCH="$1"
    shift
    DOCKER_IMAGE="$1"
+    shift

    # run it, but after, chown anything left in /tmp to *this* uid/gid, otherwise we can't delete them later...
-    docker run --rm -e ARCH -v "$(pwd)":/tmp "$DOCKER_IMAGE" sh -c "'/tmp/$BUILD_SCRIPT'; exit=\$?; chown -R '$UID:$(id -g)' /tmp; exit \$exit"
+    docker run --rm -e ARCH -e BRANCH_NAME -e BUILD_UID="$UID" -e BUILD_GID="$(id -g)" -v "$(pwd)":/tmp -w /tmp "$DOCKER_IMAGE" sh -c "\"\$@\"; exit=\$?; chown -R '$UID:$(id -g)' /tmp; exit \$exit" -- "$@"
 }

-docker_build 'amd64'   'alpine'
+docker_build 'amd64'   'alpine'                                 "$BUILD_SCRIPT" "$@"

 # before first multiarch image, must register binfmt handlers
 docker run --rm --privileged multiarch/qemu-user-static:register --reset

-docker_build 'i386'    'multiarch/alpine:i386-latest-stable'
-docker_build 'aarch64' 'multiarch/alpine:aarch64-latest-stable'
-docker_build 'armv7'   'multiarch/alpine:armv7-latest-stable'
-docker_build 'ppc64le' 'multiarch/alpine:ppc64le-latest-stable'
+docker_build 'i386'    'multiarch/alpine:i386-latest-stable'    "$BUILD_SCRIPT" "$@"
+docker_build 'aarch64' 'multiarch/alpine:aarch64-latest-stable' "$BUILD_SCRIPT" "$@"
+docker_build 'armv7'   'multiarch/alpine:armv7-latest-stable'   "$BUILD_SCRIPT" "$@"
+docker_build 'ppc64le' 'multiarch/alpine:ppc64le-latest-stable' "$BUILD_SCRIPT" "$@"
--- a/java/Dockerfile
+++ b/java/Dockerfile
@ -22,10 +22,11 @@ RUN mkdir /m2 /npm && ln -sf /m2/ /root/.m2 && ln -sf /npm/ /root/.npm && \
    curl https://download.java.net/java/early_access/jdk16/23/GPL/openjdk-16-ea+23_linux-x64_bin.tar.gz | bsdtar -xf - -C /usr/lib/jvm && \
    mv /usr/lib/jvm/java-se-9-ri/jdk-9 /usr/lib/jvm/ && rm -rf java-se-9-ri && chmod +x /usr/lib/jvm/jdk-9/bin/*

-COPY ./build.sh ./run-java /usr/bin/
+COPY ./run-java-all ./run-java /usr/bin/
+RUN ln -sf /usr/bin/run-java-all /usr/bin/build.sh

-VOLUME [ "/build", "/m2", "/npm" ]
+VOLUME [ "/build", "/m2", "/npm", "/root/.netrc" ]

 WORKDIR /build
 ENTRYPOINT ["/usr/bin/run.sh"]
-CMD ["/usr/bin/build.sh", "./.jenkins/build.sh"]
+CMD ["/usr/bin/run-java-all", "./.jenkins/build.sh"]
--- a/java/build.sh
+++ b/java/build.sh
@ -1,7 +1,9 @@
 #!/bin/bash
 set -euo pipefail

-for jdk in ${BUILD_JDKS:-6 7 8 9 10 11 12 13 14 15 16}
-do
-    run-java "$jdk" "$@"
-done
+BUILD_SCRIPT="$1"
+shift
+[ "" == "$BUILD_SCRIPT" ] && [ -f '.ci/build.sh' ] && BUILD_SCRIPT='.ci/build.sh'
+[ "" == "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh'
+
+docker run --rm -v "$HOME/.netrc:/root/.netrc:ro" -v "$HOME/.m2:/m2" -v "$HOME/.npm:/npm" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest /usr/bin/run-java-all "$BUILD_SCRIPT" "$@"
--- a/java/readme.md
+++ b/java/readme.md
@ -6,11 +6,18 @@ A docker container with every version of the Java JDK 6 to 16 installed in it, a
 Meant to be ran in CI something like this:

 ```sh
+# docker directly
 docker run --rm -v "$HOME/.m2:/m2" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest
+
+# through self-ci script hosted at code.moparisthebest.com
+curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/build-ci.sh | bash -s -- java
+
+# through self-ci script hosted at github
+curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/build-ci.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- java
 ```

 Without arguments it will execute `.jenkins/build.sh` once for each version of Java installed, setting the env variables JAVA_VERSION (a number), JAVA_HOME, M2_HOME, and PATH appropriately so invocations of `mvn` and `java` *just work*.  If you want to call another script each time:

 ```sh
-docker run --rm -v "$HOME/.m2:/m2" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest build.sh ./path/to/your/script.sh
+docker run --rm -v "$HOME/.m2:/m2" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest run-java-all ./path/to/your/script.sh
 ```
--- a/java/run-java
+++ b/java/run-java
@ -60,6 +60,6 @@ else
 fi

 # maven 3.2.5 is the latest version supported by 6
-[ $JAVA_VERSION -eq 6 ] && export M2_HOME="/opt/apache-maven-3.2.5"
+[ $JAVA_VERSION -eq 6 ] && M2_HOME="/opt/apache-maven-3.2.5"

 JAVA_VERSION=$JAVA_VERSION M2_HOME="$M2_HOME" JAVA_HOME="$JAVA_HOME" PATH="$JAVA_HOME/bin:$M2_HOME/bin:$PATH" exec "$@"
--- a/java/run-java-all
+++ b/java/run-java-all
@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+
+for jdk in ${BUILD_JDKS:-6 7 8 9 10 11 12 13 14 15 16}
+do
+    run-java "$jdk" "$@"
+done
--- a/rust/build-jenkins.sh
+++ b/rust/build-jenkins.sh
@ -1,37 +0,0 @@
-#!/bin/bash
-set -euxo pipefail
-
-# this runs first arg or .jenkins/build.sh with env variable TARGET and DISABLE_TESTS set appropriately once for each target supported by cross
-# this'll have to do until cross can run in docker containers backed by the btrfs driver...
-
-# run like:
-#   curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/rust/build-jenkins.sh | bash
-#   curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/rust/build-jenkins.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash
-# or with custom script:
-#   curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/rust/build-jenkins.sh | bash -s -- ./path/to/build.sh
-#   curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/rust/build-jenkins.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- ./path/to/build.sh
-
-export BUILD_SCRIPT="${1-.jenkins/build.sh}"
-shift
-export RELEASE_SCRIPT="${1-.jenkins/release.sh}"
-
-mkdir bin
-cd bin
-curl --compressed -sL -O https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/ci-release-helper.sh -O https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/rust/build.sh
-chmod +x ci-release-helper.sh build.sh
-cd ..
-export PATH="$(pwd)/bin:$PATH"
-
-build.sh "$BUILD_SCRIPT"
-
-if [ -e "$RELEASE_SCRIPT" ]
-then
-    "$RELEASE_SCRIPT"
-elif [ -d 'release' ]
-then
-    # default release script
-    ci-release-helper.sh standard_pre_release
-    find release/ -type f -print0 | xargs -0n1 -I {} ci-release-helper.sh standard_multi_release '{}' 'application/octet-stream'
-fi
-
-exit 0
--- a/rust/build.sh
+++ b/rust/build.sh
@ -1,14 +1,19 @@
 #!/bin/bash
 set -euo pipefail

+BUILD_SCRIPT="$1"
+shift
+[ "" == "$BUILD_SCRIPT" ] && [ -f '.ci/build.sh' ] && BUILD_SCRIPT='.ci/build.sh'
+[ "" == "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh'
+
 export CROSS_VERSION=0.2.1

 for TARGET in ${BUILD_TARGETS:-x86_64-unknown-linux-musl x86_64-unknown-linux-gnu i686-unknown-linux-musl i686-unknown-linux-gnu i586-unknown-linux-musl i586-unknown-linux-gnu aarch64-unknown-linux-musl aarch64-unknown-linux-gnu armv7-unknown-linux-gnueabihf armv7-unknown-linux-musleabihf arm-unknown-linux-gnueabi arm-unknown-linux-gnueabihf arm-unknown-linux-musleabi arm-unknown-linux-musleabihf armv5te-unknown-linux-gnueabi armv5te-unknown-linux-musleabi x86_64-pc-windows-gnu x86_64-linux-android i686-linux-android aarch64-linux-android armv7-linux-androideabi arm-linux-androideabi mips64el-unknown-linux-gnuabi64 mips64-unknown-linux-gnuabi64 mipsel-unknown-linux-gnu mipsel-unknown-linux-musl mips-unknown-linux-gnu mips-unknown-linux-musl powerpc64le-unknown-linux-gnu powerpc-unknown-linux-gnu riscv64gc-unknown-linux-gnu s390x-unknown-linux-gnu x86_64-sun-solaris sparcv9-sun-solaris x86_64-unknown-netbsd}
 do
    if echo "$TARGET" | grep -E '(^s390x|^thumb|solaris$|^x86_64-unknown-dragonfly$|^x86_64-unknown-netbsd$)' >/dev/null
    then
-        DISABLE_TESTS=1 TARGET="$TARGET" "$@"
+        DISABLE_TESTS=1 TARGET="$TARGET" "$BUILD_SCRIPT" "$@"
    else
-        TARGET="$TARGET" "$@"
+        TARGET="$TARGET" "$BUILD_SCRIPT" "$@"
    fi
 done