Browse Source

Re-factor self-ci scripts

master
Travis Burtrum 2 years ago
parent
commit
e8c711b0be
  1. 0
      .ci/Jenkinsfile
  2. 76
      build-ci.sh
  3. 27
      c/build.sh
  4. 7
      java/Dockerfile
  5. 10
      java/build.sh
  6. 9
      java/readme.md
  7. 2
      java/run-java
  8. 7
      java/run-java-all
  9. 37
      rust/build-jenkins.sh
  10. 9
      rust/build.sh

0
.jenkins/Jenkinsfile → .ci/Jenkinsfile vendored

76
build-ci.sh

@ -4,17 +4,69 @@ @@ -4,17 +4,69 @@
# 1. rust, this runs $BUILD_SCRIPT with env variable TARGET and DISABLE_TESTS set appropriately once for each target supported by cross
# this'll have to do until cross can run in docker containers backed by the btrfs driver...
# 2. c, this runs $BUILD_SCRIPT inside an alpine linux container, once for each ARCH supported by build.sh here
# 3. java, this runs $BUILD_SCRIPT inside docker container moparisthebest/self-ci-java:latest once for each version of Java currently supported
# run like (for rust):
# curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/build-ci.sh | bash -s -- rust
# curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/build-ci.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- rust
# curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/build-ci.sh | bash -s -- -l rust
# curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/build-ci.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- -l rust
export BUILD_LANG="${1-rust}"
shift
export BUILD_SCRIPT="${1-.jenkins/build.sh}"
shift
export RELEASE_SCRIPT="${1-.jenkins/release.sh}"
shift
BUILD_LANG=''
RELEASE_SCRIPT='.ci/release.sh'
BUILD_SCRIPT='.ci/build.sh'
DEFAULT_RELEASE_DIR='release'
# compat with alternative directory name
[ ! -f "$RELEASE_SCRIPT" ] && [ -f '.jenkins/release.sh' ] && RELEASE_SCRIPT='.jenkins/release.sh'
[ ! -f "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh'
function usage {
echo "$@"
echo
echo "Usage: build-ci.sh [-l BUILD_LANG] [-r RELEASE_SCRIPT] [-d DEFAULT_RELEASE_DIR] [-b BUILD_SCRIPT]" 2>&1
echo 'Build a project with self-ci.'
echo ' -l BUILD_LANG Specify the language build script to grab+execute (must be one of: rust, c, java)'
echo ' -r RELEASE_SCRIPT Specify the release script in this directory to run after build is successful'
echo ' -d DEFAULT_RELEASE_DIR If RELEASE_SCRIPT not specified, and this directory exists after successful build'
echo ' then run default release script on all files in it'
echo ' -b BUILD_SCRIPT Specify the build script in this directory to run with above language script'
exit 1
}
while getopts 'l:b:r:d:' arg; do
case ${arg} in
l)
BUILD_LANG="${OPTARG}"
[ "" == "$(echo "$BUILD_LANG" | grep -E '^(rust|c|java)$')" ] && usage "Invalid lang $BUILD_LANG, must be one of: rust, c, java"
;;
b)
BUILD_SCRIPT="${OPTARG}"
;;
r)
RELEASE_SCRIPT="${OPTARG}"
[ ! -e "$RELEASE_SCRIPT" ] && usage "Release script $RELEASE_SCRIPT does not exist"
;;
d)
DEFAULT_RELEASE_DIR="${OPTARG}"
;;
?)
usage "Invalid option: -${OPTARG}."
;;
esac
done
shift "$((OPTIND-1))"
# for all currently supported langs, this must exist in *this* directory
[ ! -e "$BUILD_SCRIPT" ] && usage "Build script $BUILD_SCRIPT does not exist"
# a bit of deterministic smarts to auto-detect BUILD_LANG
[ "$BUILD_LANG" == "" ] && [ -e pom.xml -o -e build.gradle ] && BUILD_LANG=java
[ "$BUILD_LANG" == "" ] && [ -e Cargo.toml ] && BUILD_LANG=rust
[ "$BUILD_LANG" == "" ] && [ -e Makefile ] && BUILD_LANG=c
[ "$BUILD_LANG" == "" ] && usage "-l BUILD_LANG autodetection failed, must be set manually"
echo "build-ci.sh: BUILD_LANG=$BUILD_LANG RELEASE_SCRIPT=$RELEASE_SCRIPT BUILD_SCRIPT=$BUILD_SCRIPT remaining args:" "$@"
set -euxo pipefail
@ -23,18 +75,22 @@ cd bin @@ -23,18 +75,22 @@ cd bin
curl --compressed -sL -O 'https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/ci-release-helper.sh' -O "https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/$BUILD_LANG/build.sh"
chmod +x ci-release-helper.sh build.sh
cd ..
export PATH="$(pwd)/bin:$PATH"
export BRANCH_NAME="$BRANCH_NAME"
export BUILD_UID="$UID"
export BUILD_GID="$(id -g)"
build.sh "$BUILD_SCRIPT" "$@"
if [ -e "$RELEASE_SCRIPT" ]
then
"$RELEASE_SCRIPT" "$@"
elif [ -d 'release' ]
elif [ -d "$DEFAULT_RELEASE_DIR" ]
then
# default release script
ci-release-helper.sh standard_pre_release
cd release
cd "$DEFAULT_RELEASE_DIR"
find -type f ! -path ./sha256sum.txt -print0 | xargs -0 sha256sum > sha256sum.txt
gpg --clearsign sha256sum.txt
ci-release-helper.sh standard_multi_release 'sha256sum.txt' 'text/plain'

27
c/build.sh

@ -1,23 +1,36 @@ @@ -1,23 +1,36 @@
#!/bin/sh
set -euxo pipefail
export BUILD_SCRIPT="${1-.jenkins/build.sh}"
BUILD_SCRIPT="$1"
shift
[ "" == "$BUILD_SCRIPT" ] && [ -f '.ci/build.sh' ] && BUILD_SCRIPT='.ci/build.sh'
[ "" == "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh'
# this assists in cleanup, ie if a job was terminated, to be run in a jenkins finally {} or similar
if [ "$BUILD_SCRIPT" == "docker-chown" ]
then
# chown everything in this directory to this uid/gid
docker run --rm -v "$(pwd)":/tmp alpine chown -R "$UID:$(id -g)" /tmp
exit $?
fi
docker_build() {
export ARCH="$1"
shift
DOCKER_IMAGE="$1"
shift
# run it, but after, chown anything left in /tmp to *this* uid/gid, otherwise we can't delete them later...
docker run --rm -e ARCH -v "$(pwd)":/tmp "$DOCKER_IMAGE" sh -c "'/tmp/$BUILD_SCRIPT'; exit=\$?; chown -R '$UID:$(id -g)' /tmp; exit \$exit"
docker run --rm -e ARCH -e BRANCH_NAME -e BUILD_UID="$UID" -e BUILD_GID="$(id -g)" -v "$(pwd)":/tmp -w /tmp "$DOCKER_IMAGE" sh -c "\"\$@\"; exit=\$?; chown -R '$UID:$(id -g)' /tmp; exit \$exit" -- "$@"
}
docker_build 'amd64' 'alpine'
docker_build 'amd64' 'alpine' "$BUILD_SCRIPT" "$@"
# before first multiarch image, must register binfmt handlers
docker run --rm --privileged multiarch/qemu-user-static:register --reset
docker_build 'i386' 'multiarch/alpine:i386-latest-stable'
docker_build 'aarch64' 'multiarch/alpine:aarch64-latest-stable'
docker_build 'armv7' 'multiarch/alpine:armv7-latest-stable'
docker_build 'ppc64le' 'multiarch/alpine:ppc64le-latest-stable'
docker_build 'i386' 'multiarch/alpine:i386-latest-stable' "$BUILD_SCRIPT" "$@"
docker_build 'aarch64' 'multiarch/alpine:aarch64-latest-stable' "$BUILD_SCRIPT" "$@"
docker_build 'armv7' 'multiarch/alpine:armv7-latest-stable' "$BUILD_SCRIPT" "$@"
docker_build 'ppc64le' 'multiarch/alpine:ppc64le-latest-stable' "$BUILD_SCRIPT" "$@"

7
java/Dockerfile

@ -22,10 +22,11 @@ RUN mkdir /m2 /npm && ln -sf /m2/ /root/.m2 && ln -sf /npm/ /root/.npm && \ @@ -22,10 +22,11 @@ RUN mkdir /m2 /npm && ln -sf /m2/ /root/.m2 && ln -sf /npm/ /root/.npm && \
curl https://download.java.net/java/early_access/jdk16/23/GPL/openjdk-16-ea+23_linux-x64_bin.tar.gz | bsdtar -xf - -C /usr/lib/jvm && \
mv /usr/lib/jvm/java-se-9-ri/jdk-9 /usr/lib/jvm/ && rm -rf java-se-9-ri && chmod +x /usr/lib/jvm/jdk-9/bin/*
COPY ./build.sh ./run-java /usr/bin/
COPY ./run-java-all ./run-java /usr/bin/
RUN ln -sf /usr/bin/run-java-all /usr/bin/build.sh
VOLUME [ "/build", "/m2", "/npm" ]
VOLUME [ "/build", "/m2", "/npm", "/root/.netrc" ]
WORKDIR /build
ENTRYPOINT ["/usr/bin/run.sh"]
CMD ["/usr/bin/build.sh", "./.jenkins/build.sh"]
CMD ["/usr/bin/run-java-all", "./.jenkins/build.sh"]

10
java/build.sh

@ -1,7 +1,9 @@ @@ -1,7 +1,9 @@
#!/bin/bash
set -euo pipefail
for jdk in ${BUILD_JDKS:-6 7 8 9 10 11 12 13 14 15 16}
do
run-java "$jdk" "$@"
done
BUILD_SCRIPT="$1"
shift
[ "" == "$BUILD_SCRIPT" ] && [ -f '.ci/build.sh' ] && BUILD_SCRIPT='.ci/build.sh'
[ "" == "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh'
docker run --rm -v "$HOME/.netrc:/root/.netrc:ro" -v "$HOME/.m2:/m2" -v "$HOME/.npm:/npm" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest /usr/bin/run-java-all "$BUILD_SCRIPT" "$@"

9
java/readme.md

@ -6,11 +6,18 @@ A docker container with every version of the Java JDK 6 to 16 installed in it, a @@ -6,11 +6,18 @@ A docker container with every version of the Java JDK 6 to 16 installed in it, a
Meant to be ran in CI something like this:
```sh
# docker directly
docker run --rm -v "$HOME/.m2:/m2" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest
# through self-ci script hosted at code.moparisthebest.com
curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/build-ci.sh | bash -s -- java
# through self-ci script hosted at github
curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/build-ci.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- java
```
Without arguments it will execute `.jenkins/build.sh` once for each version of Java installed, setting the env variables JAVA_VERSION (a number), JAVA_HOME, M2_HOME, and PATH appropriately so invocations of `mvn` and `java` *just work*. If you want to call another script each time:
```sh
docker run --rm -v "$HOME/.m2:/m2" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest build.sh ./path/to/your/script.sh
docker run --rm -v "$HOME/.m2:/m2" -v "$PWD:/build" -e BRANCH_NAME -e BUILD_UID=$UID -e BUILD_GID=$(id -g) moparisthebest/self-ci-java:latest run-java-all ./path/to/your/script.sh
```

2
java/run-java

@ -60,6 +60,6 @@ else @@ -60,6 +60,6 @@ else
fi
# maven 3.2.5 is the latest version supported by 6
[ $JAVA_VERSION -eq 6 ] && export M2_HOME="/opt/apache-maven-3.2.5"
[ $JAVA_VERSION -eq 6 ] && M2_HOME="/opt/apache-maven-3.2.5"
JAVA_VERSION=$JAVA_VERSION M2_HOME="$M2_HOME" JAVA_HOME="$JAVA_HOME" PATH="$JAVA_HOME/bin:$M2_HOME/bin:$PATH" exec "$@"

7
java/run-java-all

@ -0,0 +1,7 @@ @@ -0,0 +1,7 @@
#!/bin/bash
set -euo pipefail
for jdk in ${BUILD_JDKS:-6 7 8 9 10 11 12 13 14 15 16}
do
run-java "$jdk" "$@"
done

37
rust/build-jenkins.sh

@ -1,37 +0,0 @@ @@ -1,37 +0,0 @@
#!/bin/bash
set -euxo pipefail
# this runs first arg or .jenkins/build.sh with env variable TARGET and DISABLE_TESTS set appropriately once for each target supported by cross
# this'll have to do until cross can run in docker containers backed by the btrfs driver...
# run like:
# curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/rust/build-jenkins.sh | bash
# curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/rust/build-jenkins.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash
# or with custom script:
# curl --compressed -sL https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/rust/build-jenkins.sh | bash -s -- ./path/to/build.sh
# curl --compressed -sL https://raw.githubusercontent.com/moparisthebest/self-ci/master/rust/build-jenkins.sh | sed 's@https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master@https://raw.githubusercontent.com/moparisthebest/self-ci/master@g' | bash -s -- ./path/to/build.sh
export BUILD_SCRIPT="${1-.jenkins/build.sh}"
shift
export RELEASE_SCRIPT="${1-.jenkins/release.sh}"
mkdir bin
cd bin
curl --compressed -sL -O https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/ci-release-helper.sh -O https://code.moparisthebest.com/moparisthebest/self-ci/raw/branch/master/rust/build.sh
chmod +x ci-release-helper.sh build.sh
cd ..
export PATH="$(pwd)/bin:$PATH"
build.sh "$BUILD_SCRIPT"
if [ -e "$RELEASE_SCRIPT" ]
then
"$RELEASE_SCRIPT"
elif [ -d 'release' ]
then
# default release script
ci-release-helper.sh standard_pre_release
find release/ -type f -print0 | xargs -0n1 -I {} ci-release-helper.sh standard_multi_release '{}' 'application/octet-stream'
fi
exit 0

9
rust/build.sh

@ -1,14 +1,19 @@ @@ -1,14 +1,19 @@
#!/bin/bash
set -euo pipefail
BUILD_SCRIPT="$1"
shift
[ "" == "$BUILD_SCRIPT" ] && [ -f '.ci/build.sh' ] && BUILD_SCRIPT='.ci/build.sh'
[ "" == "$BUILD_SCRIPT" ] && [ -f '.jenkins/build.sh' ] && BUILD_SCRIPT='.jenkins/build.sh'
export CROSS_VERSION=0.2.1
for TARGET in ${BUILD_TARGETS:-x86_64-unknown-linux-musl x86_64-unknown-linux-gnu i686-unknown-linux-musl i686-unknown-linux-gnu i586-unknown-linux-musl i586-unknown-linux-gnu aarch64-unknown-linux-musl aarch64-unknown-linux-gnu armv7-unknown-linux-gnueabihf armv7-unknown-linux-musleabihf arm-unknown-linux-gnueabi arm-unknown-linux-gnueabihf arm-unknown-linux-musleabi arm-unknown-linux-musleabihf armv5te-unknown-linux-gnueabi armv5te-unknown-linux-musleabi x86_64-pc-windows-gnu x86_64-linux-android i686-linux-android aarch64-linux-android armv7-linux-androideabi arm-linux-androideabi mips64el-unknown-linux-gnuabi64 mips64-unknown-linux-gnuabi64 mipsel-unknown-linux-gnu mipsel-unknown-linux-musl mips-unknown-linux-gnu mips-unknown-linux-musl powerpc64le-unknown-linux-gnu powerpc-unknown-linux-gnu riscv64gc-unknown-linux-gnu s390x-unknown-linux-gnu x86_64-sun-solaris sparcv9-sun-solaris x86_64-unknown-netbsd}
do
if echo "$TARGET" | grep -E '(^s390x|^thumb|solaris$|^x86_64-unknown-dragonfly$|^x86_64-unknown-netbsd$)' >/dev/null
then
DISABLE_TESTS=1 TARGET="$TARGET" "$@"
DISABLE_TESTS=1 TARGET="$TARGET" "$BUILD_SCRIPT" "$@"
else
TARGET="$TARGET" "$@"
TARGET="$TARGET" "$BUILD_SCRIPT" "$@"
fi
done

Loading…
Cancel
Save