Merge branch 'feature/gitlab-pipeline'

.gitlab-ci.yml

@@ -0,0 +1,96 @@
+stages:
+- build
+- export
+
+"build@main":
+  image: registry.gitlab.com/xsf/docker-images/xep-buildspace/image:0.1.0
+  stage: build
+  script:
+  - python3 tools/ci-restore-timestamps.py
+  - make html inbox-html inbox-xml pdf xeplist refs xml
+  - bash tools/ci-prune-build.sh
+  rules:
+  - if: '$CI_COMMIT_REF_NAME =~ /^main$/'
+    when: always
+  - when: never
+  cache:
+    key: build-cache
+    paths:
+    - build/
+  artifacts:
+    paths:
+    - build/
+    expire_in: '1 day'
+  resource_group: xep-build
+
+"pack@main":
+  image: docker:19.03.11
+  stage: export
+  services:
+  - docker:19.03.11-dind
+  script:
+  - 'export IMAGE_REF="${CI_REGISTRY_IMAGE}/packed:main-$(date -Idate)-${CI_COMMIT_SHORT_SHA}"'
+  - 'export LATEST_REF="${CI_REGISTRY_IMAGE}/packed:main-latest"'
+  - 'docker build -t "$IMAGE_REF" -f pack-only.Dockerfile .'
+  - 'docker image tag "$IMAGE_REF" "$LATEST_REF"'
+  - 'docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY'
+  - 'docker push "$IMAGE_REF"'
+  - 'docker push "$LATEST_REF"'
+  rules:
+  - if: '$CI_COMMIT_REF_NAME =~ /^main$/'
+    when: on_success
+  - when: never
+  resource_group: xep-pack
+
+"attic@main":
+  image: python:3
+  stage: export
+  script:
+  - bash -x ./tools/ci-archive.sh
+  cache:
+    paths:
+    - state/
+    key: attic-state
+  rules:
+  - if: '$CI_COMMIT_REF_NAME =~ /^main$/'
+    when: on_success
+  - when: never
+  resource_group: xep-attic
+
+"announce@main":
+  image: python:3
+  stage: export
+  script:
+  - bash -x ./tools/ci-announce.sh
+  cache:
+    paths:
+    - state/
+    key: announce-state
+  rules:
+  - if: '$CI_COMMIT_REF_NAME =~ /^main$/'
+    when: on_success
+  - when: never
+  resource_group: xep-announce
+
+"build@mr":
+  image: registry.gitlab.com/xsf/docker-images/xep-buildspace-slim/image:0.1.1
+  stage: build
+  script:
+  - python3 tools/ci-restore-timestamps.py
+  - make html inbox-html
+  - git fetch --depth=1 origin main
+  - bash tools/ci-changed-builds.sh origin/main
+  rules:
+  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+    when: always
+  - when: never
+  cache:
+    key: build-cache
+    paths:
+    - build/
+    policy: pull
+  artifacts:
+    expose_as: "Changed Documents"
+    paths: ["rendered-changes/"]
+    expire_in: '7 days'
+

Makefile

@@ -3,7 +3,7 @@
 OUTDIR?=build
 REFSDIR?=$(OUTDIR)/refs
 EXAMPLESDIR?=$(OUTDIR)/examples
-XMLDEPS=xep.xsd xep.ent xep.dtd ref.xsl $(OUTDIR)
+XMLDEPS=xep.xsd xep.ent xep.dtd ref.xsl
 TEXMLDEPS=xep2texml.xsl $(OUTDIR)/xmpp.pdf $(OUTDIR)/xmpp-text.pdf
 XEPDIRS=. inbox
 HTMLDEPS=xep.xsl $(CSSTARGETS) $(JSTARGETS)
@@ -100,13 +100,13 @@ $(OUTDIR)/xep.xsl: xep.xsl $(OUTDIR)
 $(OUTDIR)/xeplist.xml: $(wildcard *.xml) $(wildcard inbox/*.xml)
 	./tools/extract-metadata.py > $@
 
-$(EXAMPLESDIR)/%.xml: xep-%.xml $(XMLDEPS) examples.xsl $(EXAMPLESDIR)
+$(EXAMPLESDIR)/%.xml: xep-%.xml $(XMLDEPS) examples.xsl | $(EXAMPLESDIR)
 	xsltproc --path $(CURDIR) examples.xsl "$<" > "$@" && echo "Finished building $@"
 
-$(REFSDIR)/reference.XSF.XEP-%.xml: xep-%.xml $(XMLDEPS) ref.xsl $(REFSDIR)
+$(REFSDIR)/reference.XSF.XEP-%.xml: xep-%.xml $(XMLDEPS) ref.xsl | $(REFSDIR)
 	xsltproc --path $(CURDIR) ref.xsl "$<" > "$@" && echo "Finished building $@"
 
-$(xep_htmls): $(OUTDIR)/xep-%.html: xep-%.xml $(XMLDEPS) $(HTMLDEPS)
+$(xep_htmls): $(OUTDIR)/xep-%.html: xep-%.xml $(XMLDEPS) $(HTMLDEPS) | $(OUTDIR)
 	xmllint --nonet --noout --noent --loaddtd --valid "$<"
 	# Check for non-data URIs
 	! xmllint --nonet --noout --noent --loaddtd --xpath "//img/@src[not(starts-with(., 'data:'))]" $< 2>/dev/null && true
@@ -114,7 +114,7 @@ $(xep_htmls): $(OUTDIR)/xep-%.html: xep-%.xml $(XMLDEPS) $(HTMLDEPS)
 	# Actually build the HTML
 	xsltproc --path $(CURDIR) --param htmlbase "$(if $(findstring inbox,$<),'../','./')" xep.xsl "$<" > "$@" && echo "Finished building $@"
 
-$(proto_xep_htmls): $(OUTDIR)/inbox/%.html: inbox/%.xml $(XMLDEPS) $(proto_HTMLDEPS)
+$(proto_xep_htmls): $(OUTDIR)/inbox/%.html: inbox/%.xml $(XMLDEPS) $(proto_HTMLDEPS) | $(OUTDIR)
 	xmllint --nonet --noout --noent --loaddtd --valid "$<"
 	# Check for non-data URIs
 	! xmllint --nonet --noout --noent --loaddtd --xpath "//img/@src[not(starts-with(., 'data:'))]" $< 2>/dev/null && true
@@ -122,7 +122,7 @@ $(proto_xep_htmls): $(OUTDIR)/inbox/%.html: inbox/%.xml $(XMLDEPS) $(proto_HTMLD
 	# Actually build the HTML
 	xsltproc --path $(CURDIR) --param htmlbase "$(if $(findstring inbox,$<),'../','./')" xep.xsl "$<" > "$@" && echo "Finished building $@"
 
-$(OUTDIR)/xmpp.pdf $(OUTDIR)/xmpp-text.pdf: $(OUTDIR)
+$(OUTDIR)/xmpp.pdf $(OUTDIR)/xmpp-text.pdf: | $(OUTDIR)
 	cp "resources/$(notdir $@)" "$@"
 
 $(OUTDIR)/%.pdf: %.xml $(XMLDEPS) $(TEXMLDEPS)
@@ -140,16 +140,16 @@ $(OUTDIR)/%.pdf: %.xml $(XMLDEPS) $(TEXMLDEPS)
 	done
 	echo "Finished building $@"
 
-$(JSTARGETS): $(OUTDIR)
+$(JSTARGETS): | $(OUTDIR)
 	cp "$(notdir $@)" "$@"
 
-$(CSSTARGETS): $(OUTDIR)
+$(CSSTARGETS): | $(OUTDIR)
 	cp "$(notdir $@)" "$@"
 
-$(proto_JSTARGETS): $(OUTDIR)/inbox
+$(proto_JSTARGETS): | $(OUTDIR)/inbox
 	cp "$(notdir $@)" "$@"
 
-$(proto_CSSTARGETS): $(OUTDIR)/inbox
+$(proto_CSSTARGETS): | $(OUTDIR)/inbox
 	cp "$(notdir $@)" "$@"
 
 $(EXAMPLESDIR) $(REFSDIR) $(OUTDIR) $(OUTDIR)/inbox:

pack-only.Dockerfile

@@ -0,0 +1,5 @@
+FROM nginx:1-alpine
+RUN mkdir /usr/share/nginx/html/extensions/
+COPY build/ /usr/share/nginx/html/extensions/
+RUN sed -ri '/root\s+\/usr\/share\/nginx\/html/s/^(.+)$/\1\nautoindex on;/' /etc/nginx/conf.d/default.conf
+EXPOSE 80

tools/archive.py

@@ -11,12 +11,13 @@ from datetime import datetime, timedelta
 from xeplib import load_xepinfos, Status
 
 
-def do_archive(xeps_dir, attic, xep, old_version, new_version):
+def do_archive(xeps_dir, attic, xep, old_version, new_version, build):
     curr_file = xeps_dir / "xep-{:04d}.html".format(xep)
     attic_file = attic / "xep-{:04d}-{}.html".format(xep, new_version)
 
     print("XEP-{:04d}:".format(xep), old_version, "->", new_version)
 
+    if build:
         subprocess.check_call(["make", "build/xep-{:04d}.html".format(xep)])
 
     shutil.copy(str(curr_file), str(attic_file))
@@ -55,6 +56,13 @@ def main():
         help="Path to the attic (defaults to ../xep-attic/content/)"
     )
 
+    parser.add_argument(
+        "--no-build",
+        action="store_false",
+        dest="build",
+        default=True,
+    )
+
     parser.add_argument(
         "xeps",
         nargs="*",
@@ -89,7 +97,7 @@ def main():
             continue
 
         force_archive.discard(xep)
-        do_archive(args.xeps_dir, args.attic, xep, old_version, new_version)
+        do_archive(args.xeps_dir, args.attic, xep, old_version, new_version, args.build)
         changed = True
 
     for xep in force_archive:
@@ -98,7 +106,7 @@ def main():
         )
         new_version = new_accepted[xep]["last_revision"]["version"]
 
-        do_archive(args.xeps_dir, args.attic, xep, old_version, new_version)
+        do_archive(args.xeps_dir, args.attic, xep, old_version, new_version, args.build)
         changed = True
 
     if changed:

tools/ci-announce.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+state_dir=state
+old_xeplist="$state_dir/old-xeplist.xml"
+new_xeplist="build/xeplist.xml"
+mkdir -p "$state_dir"
+
+function update_state() {
+  cp "$new_xeplist" "$old_xeplist"
+}
+
+if [ ! -f "$old_xeplist" ]; then
+  printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2
+  update_state
+  exit 0
+fi
+
+./tools/send-updates.py -y -c "$EMAIL_CFG" --no-editorial -- "$old_xeplist" "$new_xeplist" $EMAIL_RECIPIENTS
+update_state

tools/ci-archive.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -euo pipefail
+state_dir=state
+old_xeplist="$state_dir/old-xeplist.xml"
+new_xeplist="build/xeplist.xml"
+mkdir -p "$state_dir"
+
+function update_state() {
+    cp "$new_xeplist" "$old_xeplist"
+}
+
+if [ ! -f "$old_xeplist" ]; then
+    printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2
+    update_state
+    exit 0
+fi
+
+chmod 0600 "$ATTIC_ID_RSA"
+export GIT_SSH_COMMAND="ssh -i \"\$ATTIC_ID_RSA\" -o StrictHostKeyChecking=no"
+git clone git@gitlab.com:xsf/xep-attic
+python3 tools/archive.py -a xep-attic/content/ --no-build "$old_xeplist" "$new_xeplist"
+pushd xep-attic
+git add content
+git update-index --refresh
+if ! git diff-index --quiet HEAD --; then
+    git config user.name "$GIT_AUTHOR_NAME"
+    git config user.email "$GIT_AUTHOR_EMAIL"
+    git commit \
+        -m "Automated XEP build ${CI_JOB_ID}" \
+        -m "Job-URL: ${CI_JOB_URL}"
+    git push
+fi
+popd
+update_state

tools/ci-changed-builds.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n'
+filenames="$(git diff-tree -r --no-commit-id --name-only HEAD "$1" | ( grep -P '^(xep-[0-9]{4}|inbox/[^/]+)\.xml$' || true))"
+if [ -z "$filenames" ]; then
+    exit 0
+fi
+mkdir -p rendered-changes/
+cp xmpp.css prettify.css rendered-changes/
+for filename in $filenames; do
+    built_filename="build/${filename/%.xml/.html}"
+    cp "$built_filename" rendered-changes/
+done

tools/ci-prune-build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -euo pipefail
+outdir="build"
+# clean out tex build logs etc.
+find "$outdir" -type f '(' -iname "*.aux" -o -iname "*.log" -o -iname "*.toc" -o -iname "*.tex" -o -iname "*.tex.xml" -o -iname "*.out" ')' -print0 | xargs -0r -- rm
+
+find "$outdir" -type f '(' -iname "*.xml" -o -iname "*.html" -o -iname "*.pdf" ')' -print0 | while read -d $'\0' filename; do
+    if [ "$filename" = 'build/xmpp.pdf' ] || [ "$filename" = 'build/xmpp-text.pdf' ] || [ "$filename" = 'build/xeplist.xml' ]; then
+        continue
+    fi
+
+    if [[ "$filename" =~ build/refs/reference.*.xml ]]; then
+        base_filename="$(echo "$filename" | sed -r 's#^build/refs/reference\.XSF\.XEP-([0-9]+)\.xml#xep-\1.xml#')"
+    else
+        base_filename="$(echo "$filename" | sed -r 's#^build/(.+)\.[^.]+$#\1.xml#')"
+    fi
+
+    if [ ! -e "$base_filename" ]; then
+        printf 'deleting %q for which no source file (%q) exists\n' "$filename" "$base_filename"
+        rm "$filename"
+    fi
+done

tools/ci-restore-timestamps.py

@@ -0,0 +1,71 @@
+#!/usr/bin/python3
+import os
+import pathlib
+import subprocess
+import time
+
+
+def parse_timestamp_line(s: str):
+    author_ts, committer_ts = s.split(" ", 1)
+    return max(int(author_ts), int(committer_ts))
+
+
+def restore_commit_timestamps(basedir: pathlib.Path):
+    env = dict(os.environ)
+    env["LANG"] = "C.UTF-8"
+    # NOTE: the build image is still only on Python 3.4 because texml and stuff
+    # so we cannot use encoding= here and have to do decoding ourselves.
+    proc = subprocess.Popen(
+        [
+            "git", "log", "--pretty=%at %ct", "--name-status",
+        ],
+        stdout=subprocess.PIPE,
+        env=env,
+    )
+
+    seen = set()
+    last_timestamp = None
+    for line in proc.stdout:
+        if not line:
+            continue
+        if not line.endswith(b"\n"):
+            raise ValueError("line not terminated")
+        line = line[:-1].decode("utf-8")
+        if not line:
+            continue
+
+        try:
+            timestamp = parse_timestamp_line(line)
+        except ValueError:
+            pass
+        else:
+            last_timestamp = timestamp
+            continue
+
+        _, filename = line.split("\t", 1)
+        if filename in seen:
+            continue
+        seen.add(filename)
+        filepath = basedir / filename
+        try:
+            os.utime(str(filepath), (last_timestamp, last_timestamp))
+        except FileNotFoundError:
+            pass
+
+
+def main():
+    basedir = pathlib.Path.cwd()
+
+    t0 = time.monotonic()
+    try:
+        restore_commit_timestamps(basedir)
+    finally:
+        t1 = time.monotonic()
+        print("timestamp restoration took {:.2f}s".format(t1-t0))
+
+    return 0
+
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main() or 0)

tools/send-updates.py

@@ -133,6 +133,10 @@ def dummy_info(number):
     }
 
 
+def extract_version(info):
+    return info.get("last_revision", {}).get("version")
+
+
 def diff_infos(old, new):
     if old["status"] != new["status"]:
         if new["status"] == Status.PROTO:
@@ -151,8 +155,8 @@ def diff_infos(old, new):
             old["last_call"] != new["last_call"]):
         return Action.LAST_CALL
 
-    old_version = old.get("last_revision", {}).get("version")
-    new_version = new.get("last_revision", {}).get("version")
+    old_version = extract_version(old)
+    new_version = extract_version(new)
 
     if old_version != new_version:
         return Action.UPDATE
@@ -160,6 +164,32 @@ def diff_infos(old, new):
     return None
 
 
+def decompose_version(s):
+    version_info = list(s.split("."))
+    if len(version_info) < 3:
+        version_info.extend(['0'] * (3 - len(version_info)))
+    return version_info
+
+
+def filter_bump_level(old_version, new_version,
+                      include_editorial, include_non_editorial):
+    if old_version is None:
+        # treat as non-editorial
+        is_editorial = False
+    else:
+        old_version_d = decompose_version(old_version)
+        new_version_d = decompose_version(new_version)
+        # if the version number only differs in patch level or below, the change
+        # is editorial
+        is_editorial = old_version_d[:2] == new_version_d[:2]
+
+    if is_editorial and not include_editorial:
+        return False
+    if not is_editorial and not include_non_editorial:
+        return False
+    return True
+
+
 def wraptext(text):
     return "\n".join(
         itertools.chain(
@@ -262,10 +292,10 @@ def main():
     )
     parser.add_argument(
         "--no-proto",
-        dest="process_proto",
+        dest="include_protoxep",
         default=True,
         action="store_false",
-        help="Disable processing of ProtoXEPs.",
+        help="Do not announce ProtoXEPs",
     )
     parser.add_argument(
         "-n", "--dry-run",
@@ -274,6 +304,20 @@ def main():
         default=False,
         help="Instead of sending emails, print them to stdout (implies -y)",
     )
+    parser.add_argument(
+        "--no-editorial",
+        action="store_false",
+        default=True,
+        dest="include_editorial",
+        help="Do not announce editorial changes."
+    )
+    parser.add_argument(
+        "--no-non-editorial",
+        action="store_false",
+        default=True,
+        dest="include_non_editorial",
+        help="Do not announce non-editorial changes."
+    )
 
     parser.add_argument(
         "old",
@@ -334,6 +378,13 @@ def main():
         new_info = new_accepted[common_xep]
 
         action = diff_infos(old_info, new_info)
+        if action == Action.UPDATE and not filter_bump_level(
+                extract_version(old_info),
+                extract_version(new_info),
+                args.include_editorial,
+                args.include_non_editorial):
+            continue
+
         if action is not None:
             updates.append((common_xep, action, new_info))
 
@@ -345,7 +396,7 @@ def main():
         if action is not None:
             updates.append((added_xep, action, new_info))
 
-    if args.process_proto:
+    if args.include_protoxep:
         for added_proto in added_protos:
             old_info = dummy_info('xxxx')
             new_info = new_proto[added_proto]