Create GitLab.com CI pipeline

This pipeline features the following:

- Building of an nginx image with the XEPs as static files,
  in all formats.
- Incremental builds on the main branch and incremental builds
  for MRs based on the last main build.
- Automatic archiving of changed XEPs to the attic
- Automatic announcement to the mailing lists
This commit is contained in:
Jonas Schäfer 2020-06-13 13:06:09 +02:00
parent 2ea54d4a3c
commit 0dbfaff260
7 changed files with 260 additions and 0 deletions

96
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,96 @@
stages:
- build
- export
"build@main":
image: registry.gitlab.com/xsf/docker-images/xep-buildspace/image:0.1.0
stage: build
script:
- python3 tools/ci-restore-timestamps.py
- make html inbox-html inbox-xml pdf xeplist refs xml
- bash tools/ci-prune-build.sh
rules:
- if: '$CI_COMMIT_REF_NAME =~ /^main$/'
when: always
- when: never
cache:
key: build-cache
paths:
- build/
artifacts:
paths:
- build/
expire_in: '1 day'
resource_group: xep-build
"pack@main":
image: docker:19.03.11
stage: export
services:
- docker:19.03.11-dind
script:
- 'export IMAGE_REF="${CI_REGISTRY_IMAGE}/packed:main-$(date -Idate)-${CI_COMMIT_SHORT_SHA}"'
- 'export LATEST_REF="${CI_REGISTRY_IMAGE}/packed:main-latest"'
- 'docker build -t "$IMAGE_REF" -f pack-only.Dockerfile .'
- 'docker image tag "$IMAGE_REF" "$LATEST_REF"'
- 'docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY'
- 'docker push "$IMAGE_REF"'
- 'docker push "$LATEST_REF"'
rules:
- if: '$CI_COMMIT_REF_NAME =~ /^main$/'
when: on_success
- when: never
resource_group: xep-pack
"attic@main":
image: python:3
stage: export
script:
- bash -x ./tools/ci-archive.sh
cache:
paths:
- state/
key: attic-state
rules:
- if: '$CI_COMMIT_REF_NAME =~ /^main$/'
when: on_success
- when: never
resource_group: xep-attic
"announce@main":
image: python:3
stage: export
script:
- bash -x ./tools/ci-announce.sh
cache:
paths:
- state/
key: announce-state
rules:
- if: '$CI_COMMIT_REF_NAME =~ /^main$/'
when: on_success
- when: never
resource_group: xep-announce
"build@mr":
image: registry.gitlab.com/xsf/docker-images/xep-buildspace-slim/image:0.1.1
stage: build
script:
- python3 tools/ci-restore-timestamps.py
- make html inbox-html
- git fetch --depth=1 origin main
- bash tools/ci-changed-builds.sh origin/main
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
when: always
- when: never
cache:
key: build-cache
paths:
- build/
policy: pull
artifacts:
expose_as: "Changed Documents"
paths: ["rendered-changes/"]
expire_in: '7 days'

5
pack-only.Dockerfile Normal file
View File

@ -0,0 +1,5 @@
FROM nginx:1-alpine
RUN mkdir /usr/share/nginx/html/extensions/
COPY build/ /usr/share/nginx/html/extensions/
RUN sed -ri '/root\s+\/usr\/share\/nginx\/html/s/^(.+)$/\1\nautoindex on;/' /etc/nginx/conf.d/default.conf
EXPOSE 80

19
tools/ci-announce.sh Normal file
View File

@ -0,0 +1,19 @@
#!/bin/bash
set -euo pipefail
state_dir=state
old_xeplist="$state_dir/old-xeplist.xml"
new_xeplist="build/xeplist.xml"
mkdir -p "$state_dir"
function update_state() {
cp "$new_xeplist" "$old_xeplist"
}
if [ ! -f "$old_xeplist" ]; then
printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2
update_state
exit 0
fi
./tools/send-updates.py -y -c "$EMAIL_CFG" --no-editorial -- "$old_xeplist" "$new_xeplist" $EMAIL_RECIPIENTS
update_state

34
tools/ci-archive.sh Normal file
View File

@ -0,0 +1,34 @@
#!/bin/bash
set -euo pipefail
state_dir=state
old_xeplist="$state_dir/old-xeplist.xml"
new_xeplist="build/xeplist.xml"
mkdir -p "$state_dir"
function update_state() {
cp "$new_xeplist" "$old_xeplist"
}
if [ ! -f "$old_xeplist" ]; then
printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2
update_state
exit 0
fi
chmod 0600 "$ATTIC_ID_RSA"
export GIT_SSH_COMMAND="ssh -i \"\$ATTIC_ID_RSA\" -o StrictHostKeyChecking=no"
git clone git@gitlab.com:xsf/xep-attic
python3 tools/archive.py -a xep-attic/content/ --no-build "$old_xeplist" "$new_xeplist"
pushd xep-attic
git add content
git update-index --refresh
if ! git diff-index --quiet HEAD --; then
git config user.name "$GIT_AUTHOR_NAME"
git config user.email "$GIT_AUTHOR_EMAIL"
git commit \
-m "Automated XEP build ${CI_JOB_ID}" \
-m "Job-URL: ${CI_JOB_URL}"
git push
fi
popd
update_state

13
tools/ci-changed-builds.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n'
filenames="$(git diff-tree -r --no-commit-id --name-only HEAD "$1" | ( grep -P '^(xep-[0-9]{4}|inbox/[^/]+)\.xml$' || true))"
if [ -z "$filenames" ]; then
exit 0
fi
mkdir -p rendered-changes/
cp xmpp.css prettify.css rendered-changes/
for filename in $filenames; do
built_filename="build/${filename/%.xml/.html}"
cp "$built_filename" rendered-changes/
done

22
tools/ci-prune-build.sh Normal file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -euo pipefail
outdir="build"
# clean out tex build logs etc.
find "$outdir" -type f '(' -iname "*.aux" -o -iname "*.log" -o -iname "*.toc" -o -iname "*.tex" -o -iname "*.tex.xml" -o -iname "*.out" ')' -print0 | xargs -0r -- rm
find "$outdir" -type f '(' -iname "*.xml" -o -iname "*.html" -o -iname "*.pdf" ')' -print0 | while read -d $'\0' filename; do
if [ "$filename" = 'build/xmpp.pdf' ] || [ "$filename" = 'build/xmpp-text.pdf' ] || [ "$filename" = 'build/xeplist.xml' ]; then
continue
fi
if [[ "$filename" =~ build/refs/reference.*.xml ]]; then
base_filename="$(echo "$filename" | sed -r 's#^build/refs/reference\.XSF\.XEP-([0-9]+)\.xml#xep-\1.xml#')"
else
base_filename="$(echo "$filename" | sed -r 's#^build/(.+)\.[^.]+$#\1.xml#')"
fi
if [ ! -e "$base_filename" ]; then
printf 'deleting %q for which no source file (%q) exists\n' "$filename" "$base_filename"
rm "$filename"
fi
done

View File

@ -0,0 +1,71 @@
#!/usr/bin/python3
import os
import pathlib
import subprocess
import time
def parse_timestamp_line(s: str):
author_ts, committer_ts = s.split(" ", 1)
return max(int(author_ts), int(committer_ts))
def restore_commit_timestamps(basedir: pathlib.Path):
env = dict(os.environ)
env["LANG"] = "C.UTF-8"
# NOTE: the build image is still only on Python 3.4 because texml and stuff
# so we cannot use encoding= here and have to do decoding ourselves.
proc = subprocess.Popen(
[
"git", "log", "--pretty=%at %ct", "--name-status",
],
stdout=subprocess.PIPE,
env=env,
)
seen = set()
last_timestamp = None
for line in proc.stdout:
if not line:
continue
if not line.endswith(b"\n"):
raise ValueError("line not terminated")
line = line[:-1].decode("utf-8")
if not line:
continue
try:
timestamp = parse_timestamp_line(line)
except ValueError:
pass
else:
last_timestamp = timestamp
continue
_, filename = line.split("\t", 1)
if filename in seen:
continue
seen.add(filename)
filepath = basedir / filename
try:
os.utime(str(filepath), (last_timestamp, last_timestamp))
except FileNotFoundError:
pass
def main():
basedir = pathlib.Path.cwd()
t0 = time.monotonic()
try:
restore_commit_timestamps(basedir)
finally:
t1 = time.monotonic()
print("timestamp restoration took {:.2f}s".format(t1-t0))
return 0
if __name__ == "__main__":
import sys
sys.exit(main() or 0)