From a694dd21ab52492a93d7a7130da9dd10a3929fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Sch=C3=A4fer?= Date: Sat, 13 Jun 2020 14:38:11 +0200 Subject: [PATCH 1/4] Fix Makefile dependencies on directories Most jobs depend on build or one of its subdirectories. By default, this causes make to take the timestamp of the `build` directory (or the respective subdirectory) into account when calculating whether a job needs rebuilding. This is a problem, because the modified timestamp of `build` updates whenever a file is put into it. Effectively, this breaks incremental builds. Luckily, GNU(?) Make supports Order-only Dependencies, prefixed with a pipe (`|`) symbol in the dependency list. That means that the dependencies are not taken into account for freshness checks, but will be built before the target (if they are non-fresh). This commit introduces usage of Order-only Dependencies for the output directories, which fixes incremental building. --- Makefile | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 20e92320..26efb1ba 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ OUTDIR?=build REFSDIR?=$(OUTDIR)/refs EXAMPLESDIR?=$(OUTDIR)/examples -XMLDEPS=xep.xsd xep.ent xep.dtd ref.xsl $(OUTDIR) +XMLDEPS=xep.xsd xep.ent xep.dtd ref.xsl TEXMLDEPS=xep2texml.xsl $(OUTDIR)/xmpp.pdf $(OUTDIR)/xmpp-text.pdf XEPDIRS=. inbox HTMLDEPS=xep.xsl $(CSSTARGETS) $(JSTARGETS) @@ -100,13 +100,13 @@ $(OUTDIR)/xep.xsl: xep.xsl $(OUTDIR) $(OUTDIR)/xeplist.xml: $(wildcard *.xml) $(wildcard inbox/*.xml) ./tools/extract-metadata.py > $@ -$(EXAMPLESDIR)/%.xml: xep-%.xml $(XMLDEPS) examples.xsl $(EXAMPLESDIR) +$(EXAMPLESDIR)/%.xml: xep-%.xml $(XMLDEPS) examples.xsl | $(EXAMPLESDIR) xsltproc --path $(CURDIR) examples.xsl "$<" > "$@" && echo "Finished building $@" -$(REFSDIR)/reference.XSF.XEP-%.xml: xep-%.xml $(XMLDEPS) ref.xsl $(REFSDIR) +$(REFSDIR)/reference.XSF.XEP-%.xml: xep-%.xml $(XMLDEPS) ref.xsl | $(REFSDIR) xsltproc --path $(CURDIR) ref.xsl "$<" > "$@" && echo "Finished building $@" -$(xep_htmls): $(OUTDIR)/xep-%.html: xep-%.xml $(XMLDEPS) $(HTMLDEPS) +$(xep_htmls): $(OUTDIR)/xep-%.html: xep-%.xml $(XMLDEPS) $(HTMLDEPS) | $(OUTDIR) xmllint --nonet --noout --noent --loaddtd --valid "$<" # Check for non-data URIs ! xmllint --nonet --noout --noent --loaddtd --xpath "//img/@src[not(starts-with(., 'data:'))]" $< 2>/dev/null && true @@ -114,7 +114,7 @@ $(xep_htmls): $(OUTDIR)/xep-%.html: xep-%.xml $(XMLDEPS) $(HTMLDEPS) # Actually build the HTML xsltproc --path $(CURDIR) --param htmlbase "$(if $(findstring inbox,$<),'../','./')" xep.xsl "$<" > "$@" && echo "Finished building $@" -$(proto_xep_htmls): $(OUTDIR)/inbox/%.html: inbox/%.xml $(XMLDEPS) $(proto_HTMLDEPS) +$(proto_xep_htmls): $(OUTDIR)/inbox/%.html: inbox/%.xml $(XMLDEPS) $(proto_HTMLDEPS) | $(OUTDIR) xmllint --nonet --noout --noent --loaddtd --valid "$<" # Check for non-data URIs ! xmllint --nonet --noout --noent --loaddtd --xpath "//img/@src[not(starts-with(., 'data:'))]" $< 2>/dev/null && true @@ -122,7 +122,7 @@ $(proto_xep_htmls): $(OUTDIR)/inbox/%.html: inbox/%.xml $(XMLDEPS) $(proto_HTMLD # Actually build the HTML xsltproc --path $(CURDIR) --param htmlbase "$(if $(findstring inbox,$<),'../','./')" xep.xsl "$<" > "$@" && echo "Finished building $@" -$(OUTDIR)/xmpp.pdf $(OUTDIR)/xmpp-text.pdf: $(OUTDIR) +$(OUTDIR)/xmpp.pdf $(OUTDIR)/xmpp-text.pdf: | $(OUTDIR) cp "resources/$(notdir $@)" "$@" $(OUTDIR)/%.pdf: %.xml $(XMLDEPS) $(TEXMLDEPS) @@ -140,16 +140,16 @@ $(OUTDIR)/%.pdf: %.xml $(XMLDEPS) $(TEXMLDEPS) done echo "Finished building $@" -$(JSTARGETS): $(OUTDIR) +$(JSTARGETS): | $(OUTDIR) cp "$(notdir $@)" "$@" -$(CSSTARGETS): $(OUTDIR) +$(CSSTARGETS): | $(OUTDIR) cp "$(notdir $@)" "$@" -$(proto_JSTARGETS): $(OUTDIR)/inbox +$(proto_JSTARGETS): | $(OUTDIR)/inbox cp "$(notdir $@)" "$@" -$(proto_CSSTARGETS): $(OUTDIR)/inbox +$(proto_CSSTARGETS): | $(OUTDIR)/inbox cp "$(notdir $@)" "$@" $(EXAMPLESDIR) $(REFSDIR) $(OUTDIR) $(OUTDIR)/inbox: From 7eb2c2e095d6ed2f9c55ebb1a543831cd2a2b1df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Sch=C3=A4fer?= Date: Sat, 13 Jun 2020 17:08:08 +0200 Subject: [PATCH 2/4] archive.py: add support for skipping building during archiving If it is known that the documents have already been built or if it is imperative to use the versions built even if local changes have been applied since the last build, this switch comes in handy. --- tools/archive.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/archive.py b/tools/archive.py index 497ec9be..6d534fb4 100755 --- a/tools/archive.py +++ b/tools/archive.py @@ -11,13 +11,14 @@ from datetime import datetime, timedelta from xeplib import load_xepinfos, Status -def do_archive(xeps_dir, attic, xep, old_version, new_version): +def do_archive(xeps_dir, attic, xep, old_version, new_version, build): curr_file = xeps_dir / "xep-{:04d}.html".format(xep) attic_file = attic / "xep-{:04d}-{}.html".format(xep, new_version) print("XEP-{:04d}:".format(xep), old_version, "->", new_version) - subprocess.check_call(["make", "build/xep-{:04d}.html".format(xep)]) + if build: + subprocess.check_call(["make", "build/xep-{:04d}.html".format(xep)]) shutil.copy(str(curr_file), str(attic_file)) @@ -55,6 +56,13 @@ def main(): help="Path to the attic (defaults to ../xep-attic/content/)" ) + parser.add_argument( + "--no-build", + action="store_false", + dest="build", + default=True, + ) + parser.add_argument( "xeps", nargs="*", @@ -89,7 +97,7 @@ def main(): continue force_archive.discard(xep) - do_archive(args.xeps_dir, args.attic, xep, old_version, new_version) + do_archive(args.xeps_dir, args.attic, xep, old_version, new_version, args.build) changed = True for xep in force_archive: @@ -98,7 +106,7 @@ def main(): ) new_version = new_accepted[xep]["last_revision"]["version"] - do_archive(args.xeps_dir, args.attic, xep, old_version, new_version) + do_archive(args.xeps_dir, args.attic, xep, old_version, new_version, args.build) changed = True if changed: From 2ea54d4a3cc16140af4e1acf285f77430f64048c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Sch=C3=A4fer?= Date: Sun, 14 Jun 2020 11:02:57 +0200 Subject: [PATCH 3/4] send-updates.py: Introduce support to skip editorial changes We generally do not want to announce those on the mailing lists, so this switch allows us to avoid having to run without -y. --- tools/send-updates.py | 61 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/tools/send-updates.py b/tools/send-updates.py index 680d1b9c..ae38e101 100755 --- a/tools/send-updates.py +++ b/tools/send-updates.py @@ -133,6 +133,10 @@ def dummy_info(number): } +def extract_version(info): + return info.get("last_revision", {}).get("version") + + def diff_infos(old, new): if old["status"] != new["status"]: if new["status"] == Status.PROTO: @@ -151,8 +155,8 @@ def diff_infos(old, new): old["last_call"] != new["last_call"]): return Action.LAST_CALL - old_version = old.get("last_revision", {}).get("version") - new_version = new.get("last_revision", {}).get("version") + old_version = extract_version(old) + new_version = extract_version(new) if old_version != new_version: return Action.UPDATE @@ -160,6 +164,32 @@ def diff_infos(old, new): return None +def decompose_version(s): + version_info = list(s.split(".")) + if len(version_info) < 3: + version_info.extend(['0'] * (3 - len(version_info))) + return version_info + + +def filter_bump_level(old_version, new_version, + include_editorial, include_non_editorial): + if old_version is None: + # treat as non-editorial + is_editorial = False + else: + old_version_d = decompose_version(old_version) + new_version_d = decompose_version(new_version) + # if the version number only differs in patch level or below, the change + # is editorial + is_editorial = old_version_d[:2] == new_version_d[:2] + + if is_editorial and not include_editorial: + return False + if not is_editorial and not include_non_editorial: + return False + return True + + def wraptext(text): return "\n".join( itertools.chain( @@ -262,10 +292,10 @@ def main(): ) parser.add_argument( "--no-proto", - dest="process_proto", + dest="include_protoxep", default=True, action="store_false", - help="Disable processing of ProtoXEPs.", + help="Do not announce ProtoXEPs", ) parser.add_argument( "-n", "--dry-run", @@ -274,6 +304,20 @@ def main(): default=False, help="Instead of sending emails, print them to stdout (implies -y)", ) + parser.add_argument( + "--no-editorial", + action="store_false", + default=True, + dest="include_editorial", + help="Do not announce editorial changes." + ) + parser.add_argument( + "--no-non-editorial", + action="store_false", + default=True, + dest="include_non_editorial", + help="Do not announce non-editorial changes." + ) parser.add_argument( "old", @@ -334,6 +378,13 @@ def main(): new_info = new_accepted[common_xep] action = diff_infos(old_info, new_info) + if action == Action.UPDATE and not filter_bump_level( + extract_version(old_info), + extract_version(new_info), + args.include_editorial, + args.include_non_editorial): + continue + if action is not None: updates.append((common_xep, action, new_info)) @@ -345,7 +396,7 @@ def main(): if action is not None: updates.append((added_xep, action, new_info)) - if args.process_proto: + if args.include_protoxep: for added_proto in added_protos: old_info = dummy_info('xxxx') new_info = new_proto[added_proto] From 0dbfaff260c5ce8a434387ac4597531f63635aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Sch=C3=A4fer?= Date: Sat, 13 Jun 2020 13:06:09 +0200 Subject: [PATCH 4/4] Create GitLab.com CI pipeline This pipeline features the following: - Building of an nginx image with the XEPs as static files, in all formats. - Incremental builds on the main branch and incremental builds for MRs based on the last main build. - Automatic archiving of changed XEPs to the attic - Automatic announcement to the mailing lists --- .gitlab-ci.yml | 96 ++++++++++++++++++++++++++++++++++ pack-only.Dockerfile | 5 ++ tools/ci-announce.sh | 19 +++++++ tools/ci-archive.sh | 34 ++++++++++++ tools/ci-changed-builds.sh | 13 +++++ tools/ci-prune-build.sh | 22 ++++++++ tools/ci-restore-timestamps.py | 71 +++++++++++++++++++++++++ 7 files changed, 260 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 pack-only.Dockerfile create mode 100644 tools/ci-announce.sh create mode 100644 tools/ci-archive.sh create mode 100755 tools/ci-changed-builds.sh create mode 100644 tools/ci-prune-build.sh create mode 100644 tools/ci-restore-timestamps.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..e4e25a57 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,96 @@ +stages: +- build +- export + +"build@main": + image: registry.gitlab.com/xsf/docker-images/xep-buildspace/image:0.1.0 + stage: build + script: + - python3 tools/ci-restore-timestamps.py + - make html inbox-html inbox-xml pdf xeplist refs xml + - bash tools/ci-prune-build.sh + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: always + - when: never + cache: + key: build-cache + paths: + - build/ + artifacts: + paths: + - build/ + expire_in: '1 day' + resource_group: xep-build + +"pack@main": + image: docker:19.03.11 + stage: export + services: + - docker:19.03.11-dind + script: + - 'export IMAGE_REF="${CI_REGISTRY_IMAGE}/packed:main-$(date -Idate)-${CI_COMMIT_SHORT_SHA}"' + - 'export LATEST_REF="${CI_REGISTRY_IMAGE}/packed:main-latest"' + - 'docker build -t "$IMAGE_REF" -f pack-only.Dockerfile .' + - 'docker image tag "$IMAGE_REF" "$LATEST_REF"' + - 'docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY' + - 'docker push "$IMAGE_REF"' + - 'docker push "$LATEST_REF"' + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: on_success + - when: never + resource_group: xep-pack + +"attic@main": + image: python:3 + stage: export + script: + - bash -x ./tools/ci-archive.sh + cache: + paths: + - state/ + key: attic-state + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: on_success + - when: never + resource_group: xep-attic + +"announce@main": + image: python:3 + stage: export + script: + - bash -x ./tools/ci-announce.sh + cache: + paths: + - state/ + key: announce-state + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: on_success + - when: never + resource_group: xep-announce + +"build@mr": + image: registry.gitlab.com/xsf/docker-images/xep-buildspace-slim/image:0.1.1 + stage: build + script: + - python3 tools/ci-restore-timestamps.py + - make html inbox-html + - git fetch --depth=1 origin main + - bash tools/ci-changed-builds.sh origin/main + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - when: never + cache: + key: build-cache + paths: + - build/ + policy: pull + artifacts: + expose_as: "Changed Documents" + paths: ["rendered-changes/"] + expire_in: '7 days' + diff --git a/pack-only.Dockerfile b/pack-only.Dockerfile new file mode 100644 index 00000000..1a01905d --- /dev/null +++ b/pack-only.Dockerfile @@ -0,0 +1,5 @@ +FROM nginx:1-alpine +RUN mkdir /usr/share/nginx/html/extensions/ +COPY build/ /usr/share/nginx/html/extensions/ +RUN sed -ri '/root\s+\/usr\/share\/nginx\/html/s/^(.+)$/\1\nautoindex on;/' /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/tools/ci-announce.sh b/tools/ci-announce.sh new file mode 100644 index 00000000..babbfd2b --- /dev/null +++ b/tools/ci-announce.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail +state_dir=state +old_xeplist="$state_dir/old-xeplist.xml" +new_xeplist="build/xeplist.xml" +mkdir -p "$state_dir" + +function update_state() { + cp "$new_xeplist" "$old_xeplist" +} + +if [ ! -f "$old_xeplist" ]; then + printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2 + update_state + exit 0 +fi + +./tools/send-updates.py -y -c "$EMAIL_CFG" --no-editorial -- "$old_xeplist" "$new_xeplist" $EMAIL_RECIPIENTS +update_state diff --git a/tools/ci-archive.sh b/tools/ci-archive.sh new file mode 100644 index 00000000..9377074a --- /dev/null +++ b/tools/ci-archive.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail +state_dir=state +old_xeplist="$state_dir/old-xeplist.xml" +new_xeplist="build/xeplist.xml" +mkdir -p "$state_dir" + +function update_state() { + cp "$new_xeplist" "$old_xeplist" +} + +if [ ! -f "$old_xeplist" ]; then + printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2 + update_state + exit 0 +fi + +chmod 0600 "$ATTIC_ID_RSA" +export GIT_SSH_COMMAND="ssh -i \"\$ATTIC_ID_RSA\" -o StrictHostKeyChecking=no" +git clone git@gitlab.com:xsf/xep-attic +python3 tools/archive.py -a xep-attic/content/ --no-build "$old_xeplist" "$new_xeplist" +pushd xep-attic +git add content +git update-index --refresh +if ! git diff-index --quiet HEAD --; then + git config user.name "$GIT_AUTHOR_NAME" + git config user.email "$GIT_AUTHOR_EMAIL" + git commit \ + -m "Automated XEP build ${CI_JOB_ID}" \ + -m "Job-URL: ${CI_JOB_URL}" + git push +fi +popd +update_state diff --git a/tools/ci-changed-builds.sh b/tools/ci-changed-builds.sh new file mode 100755 index 00000000..d7e34829 --- /dev/null +++ b/tools/ci-changed-builds.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n' +filenames="$(git diff-tree -r --no-commit-id --name-only HEAD "$1" | ( grep -P '^(xep-[0-9]{4}|inbox/[^/]+)\.xml$' || true))" +if [ -z "$filenames" ]; then + exit 0 +fi +mkdir -p rendered-changes/ +cp xmpp.css prettify.css rendered-changes/ +for filename in $filenames; do + built_filename="build/${filename/%.xml/.html}" + cp "$built_filename" rendered-changes/ +done diff --git a/tools/ci-prune-build.sh b/tools/ci-prune-build.sh new file mode 100644 index 00000000..fdc64b72 --- /dev/null +++ b/tools/ci-prune-build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -euo pipefail +outdir="build" +# clean out tex build logs etc. +find "$outdir" -type f '(' -iname "*.aux" -o -iname "*.log" -o -iname "*.toc" -o -iname "*.tex" -o -iname "*.tex.xml" -o -iname "*.out" ')' -print0 | xargs -0r -- rm + +find "$outdir" -type f '(' -iname "*.xml" -o -iname "*.html" -o -iname "*.pdf" ')' -print0 | while read -d $'\0' filename; do + if [ "$filename" = 'build/xmpp.pdf' ] || [ "$filename" = 'build/xmpp-text.pdf' ] || [ "$filename" = 'build/xeplist.xml' ]; then + continue + fi + + if [[ "$filename" =~ build/refs/reference.*.xml ]]; then + base_filename="$(echo "$filename" | sed -r 's#^build/refs/reference\.XSF\.XEP-([0-9]+)\.xml#xep-\1.xml#')" + else + base_filename="$(echo "$filename" | sed -r 's#^build/(.+)\.[^.]+$#\1.xml#')" + fi + + if [ ! -e "$base_filename" ]; then + printf 'deleting %q for which no source file (%q) exists\n' "$filename" "$base_filename" + rm "$filename" + fi +done diff --git a/tools/ci-restore-timestamps.py b/tools/ci-restore-timestamps.py new file mode 100644 index 00000000..2b5c7296 --- /dev/null +++ b/tools/ci-restore-timestamps.py @@ -0,0 +1,71 @@ +#!/usr/bin/python3 +import os +import pathlib +import subprocess +import time + + +def parse_timestamp_line(s: str): + author_ts, committer_ts = s.split(" ", 1) + return max(int(author_ts), int(committer_ts)) + + +def restore_commit_timestamps(basedir: pathlib.Path): + env = dict(os.environ) + env["LANG"] = "C.UTF-8" + # NOTE: the build image is still only on Python 3.4 because texml and stuff + # so we cannot use encoding= here and have to do decoding ourselves. + proc = subprocess.Popen( + [ + "git", "log", "--pretty=%at %ct", "--name-status", + ], + stdout=subprocess.PIPE, + env=env, + ) + + seen = set() + last_timestamp = None + for line in proc.stdout: + if not line: + continue + if not line.endswith(b"\n"): + raise ValueError("line not terminated") + line = line[:-1].decode("utf-8") + if not line: + continue + + try: + timestamp = parse_timestamp_line(line) + except ValueError: + pass + else: + last_timestamp = timestamp + continue + + _, filename = line.split("\t", 1) + if filename in seen: + continue + seen.add(filename) + filepath = basedir / filename + try: + os.utime(str(filepath), (last_timestamp, last_timestamp)) + except FileNotFoundError: + pass + + +def main(): + basedir = pathlib.Path.cwd() + + t0 = time.monotonic() + try: + restore_commit_timestamps(basedir) + finally: + t1 = time.monotonic() + print("timestamp restoration took {:.2f}s".format(t1-t0)) + + return 0 + + +if __name__ == "__main__": + import sys + sys.exit(main() or 0)