From 0dbfaff260c5ce8a434387ac4597531f63635aa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20Sch=C3=A4fer?= <j.wielicki@sotecware.net>
Date: Sat, 13 Jun 2020 13:06:09 +0200
Subject: [PATCH] Create GitLab.com CI pipeline

This pipeline features the following:

- Building of an nginx image with the XEPs as static files,
  in all formats.
- Incremental builds on the main branch and incremental builds
  for MRs based on the last main build.
- Automatic archiving of changed XEPs to the attic
- Automatic announcement to the mailing lists
---
 .gitlab-ci.yml                 | 96 ++++++++++++++++++++++++++++++++++
 pack-only.Dockerfile           |  5 ++
 tools/ci-announce.sh           | 19 +++++++
 tools/ci-archive.sh            | 34 ++++++++++++
 tools/ci-changed-builds.sh     | 13 +++++
 tools/ci-prune-build.sh        | 22 ++++++++
 tools/ci-restore-timestamps.py | 71 +++++++++++++++++++++++++
 7 files changed, 260 insertions(+)
 create mode 100644 .gitlab-ci.yml
 create mode 100644 pack-only.Dockerfile
 create mode 100644 tools/ci-announce.sh
 create mode 100644 tools/ci-archive.sh
 create mode 100755 tools/ci-changed-builds.sh
 create mode 100644 tools/ci-prune-build.sh
 create mode 100644 tools/ci-restore-timestamps.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 00000000..e4e25a57
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,96 @@
+stages:
+- build
+- export
+
+"build@main":
+  image: registry.gitlab.com/xsf/docker-images/xep-buildspace/image:0.1.0
+  stage: build
+  script:
+  - python3 tools/ci-restore-timestamps.py
+  - make html inbox-html inbox-xml pdf xeplist refs xml
+  - bash tools/ci-prune-build.sh
+  rules:
+  - if: '$CI_COMMIT_REF_NAME =~ /^main$/'
+    when: always
+  - when: never
+  cache:
+    key: build-cache
+    paths:
+    - build/
+  artifacts:
+    paths:
+    - build/
+    expire_in: '1 day'
+  resource_group: xep-build
+
+"pack@main":
+  image: docker:19.03.11
+  stage: export
+  services:
+  - docker:19.03.11-dind
+  script:
+  - 'export IMAGE_REF="${CI_REGISTRY_IMAGE}/packed:main-$(date -Idate)-${CI_COMMIT_SHORT_SHA}"'
+  - 'export LATEST_REF="${CI_REGISTRY_IMAGE}/packed:main-latest"'
+  - 'docker build -t "$IMAGE_REF" -f pack-only.Dockerfile .'
+  - 'docker image tag "$IMAGE_REF" "$LATEST_REF"'
+  - 'docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY'
+  - 'docker push "$IMAGE_REF"'
+  - 'docker push "$LATEST_REF"'
+  rules:
+  - if: '$CI_COMMIT_REF_NAME =~ /^main$/'
+    when: on_success
+  - when: never
+  resource_group: xep-pack
+
+"attic@main":
+  image: python:3
+  stage: export
+  script:
+  - bash -x ./tools/ci-archive.sh
+  cache:
+    paths:
+    - state/
+    key: attic-state
+  rules:
+  - if: '$CI_COMMIT_REF_NAME =~ /^main$/'
+    when: on_success
+  - when: never
+  resource_group: xep-attic
+
+"announce@main":
+  image: python:3
+  stage: export
+  script:
+  - bash -x ./tools/ci-announce.sh
+  cache:
+    paths:
+    - state/
+    key: announce-state
+  rules:
+  - if: '$CI_COMMIT_REF_NAME =~ /^main$/'
+    when: on_success
+  - when: never
+  resource_group: xep-announce
+
+"build@mr":
+  image: registry.gitlab.com/xsf/docker-images/xep-buildspace-slim/image:0.1.1
+  stage: build
+  script:
+  - python3 tools/ci-restore-timestamps.py
+  - make html inbox-html
+  - git fetch --depth=1 origin main
+  - bash tools/ci-changed-builds.sh origin/main
+  rules:
+  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    when: always
+  - when: never
+  cache:
+    key: build-cache
+    paths:
+    - build/
+    policy: pull
+  artifacts:
+    expose_as: "Changed Documents"
+    paths: ["rendered-changes/"]
+    expire_in: '7 days'
+
diff --git a/pack-only.Dockerfile b/pack-only.Dockerfile
new file mode 100644
index 00000000..1a01905d
--- /dev/null
+++ b/pack-only.Dockerfile
@@ -0,0 +1,5 @@
+FROM nginx:1-alpine
+RUN mkdir /usr/share/nginx/html/extensions/
+COPY build/ /usr/share/nginx/html/extensions/
+RUN sed -ri '/root\s+\/usr\/share\/nginx\/html/s/^(.+)$/\1\nautoindex on;/' /etc/nginx/conf.d/default.conf
+EXPOSE 80
diff --git a/tools/ci-announce.sh b/tools/ci-announce.sh
new file mode 100644
index 00000000..babbfd2b
--- /dev/null
+++ b/tools/ci-announce.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+state_dir=state
+old_xeplist="$state_dir/old-xeplist.xml"
+new_xeplist="build/xeplist.xml"
+mkdir -p "$state_dir"
+
+function update_state() {
+  cp "$new_xeplist" "$old_xeplist"
+}
+
+if [ ! -f "$old_xeplist" ]; then
+  printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2
+  update_state
+  exit 0
+fi
+
+./tools/send-updates.py -y -c "$EMAIL_CFG" --no-editorial -- "$old_xeplist" "$new_xeplist" $EMAIL_RECIPIENTS
+update_state
diff --git a/tools/ci-archive.sh b/tools/ci-archive.sh
new file mode 100644
index 00000000..9377074a
--- /dev/null
+++ b/tools/ci-archive.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -euo pipefail
+state_dir=state
+old_xeplist="$state_dir/old-xeplist.xml"
+new_xeplist="build/xeplist.xml"
+mkdir -p "$state_dir"
+
+function update_state() {
+    cp "$new_xeplist" "$old_xeplist"
+}
+
+if [ ! -f "$old_xeplist" ]; then
+    printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2
+    update_state
+    exit 0
+fi
+
+chmod 0600 "$ATTIC_ID_RSA"
+export GIT_SSH_COMMAND="ssh -i \"\$ATTIC_ID_RSA\" -o StrictHostKeyChecking=no"
+git clone git@gitlab.com:xsf/xep-attic
+python3 tools/archive.py -a xep-attic/content/ --no-build "$old_xeplist" "$new_xeplist"
+pushd xep-attic
+git add content
+git update-index --refresh
+if ! git diff-index --quiet HEAD --; then
+    git config user.name "$GIT_AUTHOR_NAME"
+    git config user.email "$GIT_AUTHOR_EMAIL"
+    git commit \
+        -m "Automated XEP build ${CI_JOB_ID}" \
+        -m "Job-URL: ${CI_JOB_URL}"
+    git push
+fi
+popd
+update_state
diff --git a/tools/ci-changed-builds.sh b/tools/ci-changed-builds.sh
new file mode 100755
index 00000000..d7e34829
--- /dev/null
+++ b/tools/ci-changed-builds.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n'
+filenames="$(git diff-tree -r --no-commit-id --name-only HEAD "$1" | ( grep -P '^(xep-[0-9]{4}|inbox/[^/]+)\.xml$' || true))"
+if [ -z "$filenames" ]; then
+    exit 0
+fi
+mkdir -p rendered-changes/
+cp xmpp.css prettify.css rendered-changes/
+for filename in $filenames; do
+    built_filename="build/${filename/%.xml/.html}"
+    cp "$built_filename" rendered-changes/
+done
diff --git a/tools/ci-prune-build.sh b/tools/ci-prune-build.sh
new file mode 100644
index 00000000..fdc64b72
--- /dev/null
+++ b/tools/ci-prune-build.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+set -euo pipefail
+outdir="build"
+# clean out tex build logs etc.
+find "$outdir" -type f '(' -iname "*.aux" -o -iname "*.log" -o -iname "*.toc" -o -iname "*.tex" -o -iname "*.tex.xml" -o -iname "*.out" ')' -print0 | xargs -0r -- rm
+
+find "$outdir" -type f '(' -iname "*.xml" -o -iname "*.html" -o -iname "*.pdf" ')' -print0 | while read -d $'\0' filename; do
+    if [ "$filename" = 'build/xmpp.pdf' ] || [ "$filename" = 'build/xmpp-text.pdf' ] || [ "$filename" = 'build/xeplist.xml' ]; then
+        continue
+    fi
+
+    if [[ "$filename" =~ build/refs/reference.*.xml ]]; then
+        base_filename="$(echo "$filename" | sed -r 's#^build/refs/reference\.XSF\.XEP-([0-9]+)\.xml#xep-\1.xml#')"
+    else
+        base_filename="$(echo "$filename" | sed -r 's#^build/(.+)\.[^.]+$#\1.xml#')"
+    fi
+
+    if [ ! -e "$base_filename" ]; then
+        printf 'deleting %q for which no source file (%q) exists\n' "$filename" "$base_filename"
+        rm "$filename"
+    fi
+done
diff --git a/tools/ci-restore-timestamps.py b/tools/ci-restore-timestamps.py
new file mode 100644
index 00000000..2b5c7296
--- /dev/null
+++ b/tools/ci-restore-timestamps.py
@@ -0,0 +1,71 @@
+#!/usr/bin/python3
+import os
+import pathlib
+import subprocess
+import time
+
+
+def parse_timestamp_line(s: str):
+    author_ts, committer_ts = s.split(" ", 1)
+    return max(int(author_ts), int(committer_ts))
+
+
+def restore_commit_timestamps(basedir: pathlib.Path):
+    env = dict(os.environ)
+    env["LANG"] = "C.UTF-8"
+    # NOTE: the build image is still only on Python 3.4 because texml and stuff
+    # so we cannot use encoding= here and have to do decoding ourselves.
+    proc = subprocess.Popen(
+        [
+            "git", "log", "--pretty=%at %ct", "--name-status",
+        ],
+        stdout=subprocess.PIPE,
+        env=env,
+    )
+
+    seen = set()
+    last_timestamp = None
+    for line in proc.stdout:
+        if not line:
+            continue
+        if not line.endswith(b"\n"):
+            raise ValueError("line not terminated")
+        line = line[:-1].decode("utf-8")
+        if not line:
+            continue
+
+        try:
+            timestamp = parse_timestamp_line(line)
+        except ValueError:
+            pass
+        else:
+            last_timestamp = timestamp
+            continue
+
+        _, filename = line.split("\t", 1)
+        if filename in seen:
+            continue
+        seen.add(filename)
+        filepath = basedir / filename
+        try:
+            os.utime(str(filepath), (last_timestamp, last_timestamp))
+        except FileNotFoundError:
+            pass
+
+
+def main():
+    basedir = pathlib.Path.cwd()
+
+    t0 = time.monotonic()
+    try:
+        restore_commit_timestamps(basedir)
+    finally:
+        t1 = time.monotonic()
+        print("timestamp restoration took {:.2f}s".format(t1-t0))
+
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main() or 0)