diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..e4e25a57 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,96 @@ +stages: +- build +- export + +"build@main": + image: registry.gitlab.com/xsf/docker-images/xep-buildspace/image:0.1.0 + stage: build + script: + - python3 tools/ci-restore-timestamps.py + - make html inbox-html inbox-xml pdf xeplist refs xml + - bash tools/ci-prune-build.sh + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: always + - when: never + cache: + key: build-cache + paths: + - build/ + artifacts: + paths: + - build/ + expire_in: '1 day' + resource_group: xep-build + +"pack@main": + image: docker:19.03.11 + stage: export + services: + - docker:19.03.11-dind + script: + - 'export IMAGE_REF="${CI_REGISTRY_IMAGE}/packed:main-$(date -Idate)-${CI_COMMIT_SHORT_SHA}"' + - 'export LATEST_REF="${CI_REGISTRY_IMAGE}/packed:main-latest"' + - 'docker build -t "$IMAGE_REF" -f pack-only.Dockerfile .' + - 'docker image tag "$IMAGE_REF" "$LATEST_REF"' + - 'docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY' + - 'docker push "$IMAGE_REF"' + - 'docker push "$LATEST_REF"' + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: on_success + - when: never + resource_group: xep-pack + +"attic@main": + image: python:3 + stage: export + script: + - bash -x ./tools/ci-archive.sh + cache: + paths: + - state/ + key: attic-state + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: on_success + - when: never + resource_group: xep-attic + +"announce@main": + image: python:3 + stage: export + script: + - bash -x ./tools/ci-announce.sh + cache: + paths: + - state/ + key: announce-state + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: on_success + - when: never + resource_group: xep-announce + +"build@mr": + image: registry.gitlab.com/xsf/docker-images/xep-buildspace-slim/image:0.1.1 + stage: build + script: + - python3 tools/ci-restore-timestamps.py + - make html inbox-html + - git fetch --depth=1 origin main + - bash tools/ci-changed-builds.sh origin/main + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - when: never + cache: + key: build-cache + paths: + - build/ + policy: pull + artifacts: + expose_as: "Changed Documents" + paths: ["rendered-changes/"] + expire_in: '7 days' + diff --git a/pack-only.Dockerfile b/pack-only.Dockerfile new file mode 100644 index 00000000..1a01905d --- /dev/null +++ b/pack-only.Dockerfile @@ -0,0 +1,5 @@ +FROM nginx:1-alpine +RUN mkdir /usr/share/nginx/html/extensions/ +COPY build/ /usr/share/nginx/html/extensions/ +RUN sed -ri '/root\s+\/usr\/share\/nginx\/html/s/^(.+)$/\1\nautoindex on;/' /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/tools/ci-announce.sh b/tools/ci-announce.sh new file mode 100644 index 00000000..babbfd2b --- /dev/null +++ b/tools/ci-announce.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail +state_dir=state +old_xeplist="$state_dir/old-xeplist.xml" +new_xeplist="build/xeplist.xml" +mkdir -p "$state_dir" + +function update_state() { + cp "$new_xeplist" "$old_xeplist" +} + +if [ ! -f "$old_xeplist" ]; then + printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2 + update_state + exit 0 +fi + +./tools/send-updates.py -y -c "$EMAIL_CFG" --no-editorial -- "$old_xeplist" "$new_xeplist" $EMAIL_RECIPIENTS +update_state diff --git a/tools/ci-archive.sh b/tools/ci-archive.sh new file mode 100644 index 00000000..9377074a --- /dev/null +++ b/tools/ci-archive.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail +state_dir=state +old_xeplist="$state_dir/old-xeplist.xml" +new_xeplist="build/xeplist.xml" +mkdir -p "$state_dir" + +function update_state() { + cp "$new_xeplist" "$old_xeplist" +} + +if [ ! -f "$old_xeplist" ]; then + printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2 + update_state + exit 0 +fi + +chmod 0600 "$ATTIC_ID_RSA" +export GIT_SSH_COMMAND="ssh -i \"\$ATTIC_ID_RSA\" -o StrictHostKeyChecking=no" +git clone git@gitlab.com:xsf/xep-attic +python3 tools/archive.py -a xep-attic/content/ --no-build "$old_xeplist" "$new_xeplist" +pushd xep-attic +git add content +git update-index --refresh +if ! git diff-index --quiet HEAD --; then + git config user.name "$GIT_AUTHOR_NAME" + git config user.email "$GIT_AUTHOR_EMAIL" + git commit \ + -m "Automated XEP build ${CI_JOB_ID}" \ + -m "Job-URL: ${CI_JOB_URL}" + git push +fi +popd +update_state diff --git a/tools/ci-changed-builds.sh b/tools/ci-changed-builds.sh new file mode 100755 index 00000000..d7e34829 --- /dev/null +++ b/tools/ci-changed-builds.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n' +filenames="$(git diff-tree -r --no-commit-id --name-only HEAD "$1" | ( grep -P '^(xep-[0-9]{4}|inbox/[^/]+)\.xml$' || true))" +if [ -z "$filenames" ]; then + exit 0 +fi +mkdir -p rendered-changes/ +cp xmpp.css prettify.css rendered-changes/ +for filename in $filenames; do + built_filename="build/${filename/%.xml/.html}" + cp "$built_filename" rendered-changes/ +done diff --git a/tools/ci-prune-build.sh b/tools/ci-prune-build.sh new file mode 100644 index 00000000..fdc64b72 --- /dev/null +++ b/tools/ci-prune-build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -euo pipefail +outdir="build" +# clean out tex build logs etc. +find "$outdir" -type f '(' -iname "*.aux" -o -iname "*.log" -o -iname "*.toc" -o -iname "*.tex" -o -iname "*.tex.xml" -o -iname "*.out" ')' -print0 | xargs -0r -- rm + +find "$outdir" -type f '(' -iname "*.xml" -o -iname "*.html" -o -iname "*.pdf" ')' -print0 | while read -d $'\0' filename; do + if [ "$filename" = 'build/xmpp.pdf' ] || [ "$filename" = 'build/xmpp-text.pdf' ] || [ "$filename" = 'build/xeplist.xml' ]; then + continue + fi + + if [[ "$filename" =~ build/refs/reference.*.xml ]]; then + base_filename="$(echo "$filename" | sed -r 's#^build/refs/reference\.XSF\.XEP-([0-9]+)\.xml#xep-\1.xml#')" + else + base_filename="$(echo "$filename" | sed -r 's#^build/(.+)\.[^.]+$#\1.xml#')" + fi + + if [ ! -e "$base_filename" ]; then + printf 'deleting %q for which no source file (%q) exists\n' "$filename" "$base_filename" + rm "$filename" + fi +done diff --git a/tools/ci-restore-timestamps.py b/tools/ci-restore-timestamps.py new file mode 100644 index 00000000..2b5c7296 --- /dev/null +++ b/tools/ci-restore-timestamps.py @@ -0,0 +1,71 @@ +#!/usr/bin/python3 +import os +import pathlib +import subprocess +import time + + +def parse_timestamp_line(s: str): + author_ts, committer_ts = s.split(" ", 1) + return max(int(author_ts), int(committer_ts)) + + +def restore_commit_timestamps(basedir: pathlib.Path): + env = dict(os.environ) + env["LANG"] = "C.UTF-8" + # NOTE: the build image is still only on Python 3.4 because texml and stuff + # so we cannot use encoding= here and have to do decoding ourselves. + proc = subprocess.Popen( + [ + "git", "log", "--pretty=%at %ct", "--name-status", + ], + stdout=subprocess.PIPE, + env=env, + ) + + seen = set() + last_timestamp = None + for line in proc.stdout: + if not line: + continue + if not line.endswith(b"\n"): + raise ValueError("line not terminated") + line = line[:-1].decode("utf-8") + if not line: + continue + + try: + timestamp = parse_timestamp_line(line) + except ValueError: + pass + else: + last_timestamp = timestamp + continue + + _, filename = line.split("\t", 1) + if filename in seen: + continue + seen.add(filename) + filepath = basedir / filename + try: + os.utime(str(filepath), (last_timestamp, last_timestamp)) + except FileNotFoundError: + pass + + +def main(): + basedir = pathlib.Path.cwd() + + t0 = time.monotonic() + try: + restore_commit_timestamps(basedir) + finally: + t1 = time.monotonic() + print("timestamp restoration took {:.2f}s".format(t1-t0)) + + return 0 + + +if __name__ == "__main__": + import sys + sys.exit(main() or 0)