diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..e4e25a57 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,96 @@ +stages: +- build +- export + +"build@main": + image: registry.gitlab.com/xsf/docker-images/xep-buildspace/image:0.1.0 + stage: build + script: + - python3 tools/ci-restore-timestamps.py + - make html inbox-html inbox-xml pdf xeplist refs xml + - bash tools/ci-prune-build.sh + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: always + - when: never + cache: + key: build-cache + paths: + - build/ + artifacts: + paths: + - build/ + expire_in: '1 day' + resource_group: xep-build + +"pack@main": + image: docker:19.03.11 + stage: export + services: + - docker:19.03.11-dind + script: + - 'export IMAGE_REF="${CI_REGISTRY_IMAGE}/packed:main-$(date -Idate)-${CI_COMMIT_SHORT_SHA}"' + - 'export LATEST_REF="${CI_REGISTRY_IMAGE}/packed:main-latest"' + - 'docker build -t "$IMAGE_REF" -f pack-only.Dockerfile .' + - 'docker image tag "$IMAGE_REF" "$LATEST_REF"' + - 'docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY' + - 'docker push "$IMAGE_REF"' + - 'docker push "$LATEST_REF"' + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: on_success + - when: never + resource_group: xep-pack + +"attic@main": + image: python:3 + stage: export + script: + - bash -x ./tools/ci-archive.sh + cache: + paths: + - state/ + key: attic-state + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: on_success + - when: never + resource_group: xep-attic + +"announce@main": + image: python:3 + stage: export + script: + - bash -x ./tools/ci-announce.sh + cache: + paths: + - state/ + key: announce-state + rules: + - if: '$CI_COMMIT_REF_NAME =~ /^main$/' + when: on_success + - when: never + resource_group: xep-announce + +"build@mr": + image: registry.gitlab.com/xsf/docker-images/xep-buildspace-slim/image:0.1.1 + stage: build + script: + - python3 tools/ci-restore-timestamps.py + - make html inbox-html + - git fetch --depth=1 origin main + - bash tools/ci-changed-builds.sh origin/main + rules: + - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' + when: always + - when: never + cache: + key: build-cache + paths: + - build/ + policy: pull + artifacts: + expose_as: "Changed Documents" + paths: ["rendered-changes/"] + expire_in: '7 days' + diff --git a/Makefile b/Makefile index 20e92320..26efb1ba 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ OUTDIR?=build REFSDIR?=$(OUTDIR)/refs EXAMPLESDIR?=$(OUTDIR)/examples -XMLDEPS=xep.xsd xep.ent xep.dtd ref.xsl $(OUTDIR) +XMLDEPS=xep.xsd xep.ent xep.dtd ref.xsl TEXMLDEPS=xep2texml.xsl $(OUTDIR)/xmpp.pdf $(OUTDIR)/xmpp-text.pdf XEPDIRS=. inbox HTMLDEPS=xep.xsl $(CSSTARGETS) $(JSTARGETS) @@ -100,13 +100,13 @@ $(OUTDIR)/xep.xsl: xep.xsl $(OUTDIR) $(OUTDIR)/xeplist.xml: $(wildcard *.xml) $(wildcard inbox/*.xml) ./tools/extract-metadata.py > $@ -$(EXAMPLESDIR)/%.xml: xep-%.xml $(XMLDEPS) examples.xsl $(EXAMPLESDIR) +$(EXAMPLESDIR)/%.xml: xep-%.xml $(XMLDEPS) examples.xsl | $(EXAMPLESDIR) xsltproc --path $(CURDIR) examples.xsl "$<" > "$@" && echo "Finished building $@" -$(REFSDIR)/reference.XSF.XEP-%.xml: xep-%.xml $(XMLDEPS) ref.xsl $(REFSDIR) +$(REFSDIR)/reference.XSF.XEP-%.xml: xep-%.xml $(XMLDEPS) ref.xsl | $(REFSDIR) xsltproc --path $(CURDIR) ref.xsl "$<" > "$@" && echo "Finished building $@" -$(xep_htmls): $(OUTDIR)/xep-%.html: xep-%.xml $(XMLDEPS) $(HTMLDEPS) +$(xep_htmls): $(OUTDIR)/xep-%.html: xep-%.xml $(XMLDEPS) $(HTMLDEPS) | $(OUTDIR) xmllint --nonet --noout --noent --loaddtd --valid "$<" # Check for non-data URIs ! xmllint --nonet --noout --noent --loaddtd --xpath "//img/@src[not(starts-with(., 'data:'))]" $< 2>/dev/null && true @@ -114,7 +114,7 @@ $(xep_htmls): $(OUTDIR)/xep-%.html: xep-%.xml $(XMLDEPS) $(HTMLDEPS) # Actually build the HTML xsltproc --path $(CURDIR) --param htmlbase "$(if $(findstring inbox,$<),'../','./')" xep.xsl "$<" > "$@" && echo "Finished building $@" -$(proto_xep_htmls): $(OUTDIR)/inbox/%.html: inbox/%.xml $(XMLDEPS) $(proto_HTMLDEPS) +$(proto_xep_htmls): $(OUTDIR)/inbox/%.html: inbox/%.xml $(XMLDEPS) $(proto_HTMLDEPS) | $(OUTDIR) xmllint --nonet --noout --noent --loaddtd --valid "$<" # Check for non-data URIs ! xmllint --nonet --noout --noent --loaddtd --xpath "//img/@src[not(starts-with(., 'data:'))]" $< 2>/dev/null && true @@ -122,7 +122,7 @@ $(proto_xep_htmls): $(OUTDIR)/inbox/%.html: inbox/%.xml $(XMLDEPS) $(proto_HTMLD # Actually build the HTML xsltproc --path $(CURDIR) --param htmlbase "$(if $(findstring inbox,$<),'../','./')" xep.xsl "$<" > "$@" && echo "Finished building $@" -$(OUTDIR)/xmpp.pdf $(OUTDIR)/xmpp-text.pdf: $(OUTDIR) +$(OUTDIR)/xmpp.pdf $(OUTDIR)/xmpp-text.pdf: | $(OUTDIR) cp "resources/$(notdir $@)" "$@" $(OUTDIR)/%.pdf: %.xml $(XMLDEPS) $(TEXMLDEPS) @@ -140,16 +140,16 @@ $(OUTDIR)/%.pdf: %.xml $(XMLDEPS) $(TEXMLDEPS) done echo "Finished building $@" -$(JSTARGETS): $(OUTDIR) +$(JSTARGETS): | $(OUTDIR) cp "$(notdir $@)" "$@" -$(CSSTARGETS): $(OUTDIR) +$(CSSTARGETS): | $(OUTDIR) cp "$(notdir $@)" "$@" -$(proto_JSTARGETS): $(OUTDIR)/inbox +$(proto_JSTARGETS): | $(OUTDIR)/inbox cp "$(notdir $@)" "$@" -$(proto_CSSTARGETS): $(OUTDIR)/inbox +$(proto_CSSTARGETS): | $(OUTDIR)/inbox cp "$(notdir $@)" "$@" $(EXAMPLESDIR) $(REFSDIR) $(OUTDIR) $(OUTDIR)/inbox: diff --git a/pack-only.Dockerfile b/pack-only.Dockerfile new file mode 100644 index 00000000..1a01905d --- /dev/null +++ b/pack-only.Dockerfile @@ -0,0 +1,5 @@ +FROM nginx:1-alpine +RUN mkdir /usr/share/nginx/html/extensions/ +COPY build/ /usr/share/nginx/html/extensions/ +RUN sed -ri '/root\s+\/usr\/share\/nginx\/html/s/^(.+)$/\1\nautoindex on;/' /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/tools/archive.py b/tools/archive.py index 497ec9be..6d534fb4 100755 --- a/tools/archive.py +++ b/tools/archive.py @@ -11,13 +11,14 @@ from datetime import datetime, timedelta from xeplib import load_xepinfos, Status -def do_archive(xeps_dir, attic, xep, old_version, new_version): +def do_archive(xeps_dir, attic, xep, old_version, new_version, build): curr_file = xeps_dir / "xep-{:04d}.html".format(xep) attic_file = attic / "xep-{:04d}-{}.html".format(xep, new_version) print("XEP-{:04d}:".format(xep), old_version, "->", new_version) - subprocess.check_call(["make", "build/xep-{:04d}.html".format(xep)]) + if build: + subprocess.check_call(["make", "build/xep-{:04d}.html".format(xep)]) shutil.copy(str(curr_file), str(attic_file)) @@ -55,6 +56,13 @@ def main(): help="Path to the attic (defaults to ../xep-attic/content/)" ) + parser.add_argument( + "--no-build", + action="store_false", + dest="build", + default=True, + ) + parser.add_argument( "xeps", nargs="*", @@ -89,7 +97,7 @@ def main(): continue force_archive.discard(xep) - do_archive(args.xeps_dir, args.attic, xep, old_version, new_version) + do_archive(args.xeps_dir, args.attic, xep, old_version, new_version, args.build) changed = True for xep in force_archive: @@ -98,7 +106,7 @@ def main(): ) new_version = new_accepted[xep]["last_revision"]["version"] - do_archive(args.xeps_dir, args.attic, xep, old_version, new_version) + do_archive(args.xeps_dir, args.attic, xep, old_version, new_version, args.build) changed = True if changed: diff --git a/tools/ci-announce.sh b/tools/ci-announce.sh new file mode 100644 index 00000000..babbfd2b --- /dev/null +++ b/tools/ci-announce.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail +state_dir=state +old_xeplist="$state_dir/old-xeplist.xml" +new_xeplist="build/xeplist.xml" +mkdir -p "$state_dir" + +function update_state() { + cp "$new_xeplist" "$old_xeplist" +} + +if [ ! -f "$old_xeplist" ]; then + printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2 + update_state + exit 0 +fi + +./tools/send-updates.py -y -c "$EMAIL_CFG" --no-editorial -- "$old_xeplist" "$new_xeplist" $EMAIL_RECIPIENTS +update_state diff --git a/tools/ci-archive.sh b/tools/ci-archive.sh new file mode 100644 index 00000000..9377074a --- /dev/null +++ b/tools/ci-archive.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail +state_dir=state +old_xeplist="$state_dir/old-xeplist.xml" +new_xeplist="build/xeplist.xml" +mkdir -p "$state_dir" + +function update_state() { + cp "$new_xeplist" "$old_xeplist" +} + +if [ ! -f "$old_xeplist" ]; then + printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2 + update_state + exit 0 +fi + +chmod 0600 "$ATTIC_ID_RSA" +export GIT_SSH_COMMAND="ssh -i \"\$ATTIC_ID_RSA\" -o StrictHostKeyChecking=no" +git clone git@gitlab.com:xsf/xep-attic +python3 tools/archive.py -a xep-attic/content/ --no-build "$old_xeplist" "$new_xeplist" +pushd xep-attic +git add content +git update-index --refresh +if ! git diff-index --quiet HEAD --; then + git config user.name "$GIT_AUTHOR_NAME" + git config user.email "$GIT_AUTHOR_EMAIL" + git commit \ + -m "Automated XEP build ${CI_JOB_ID}" \ + -m "Job-URL: ${CI_JOB_URL}" + git push +fi +popd +update_state diff --git a/tools/ci-changed-builds.sh b/tools/ci-changed-builds.sh new file mode 100755 index 00000000..d7e34829 --- /dev/null +++ b/tools/ci-changed-builds.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n' +filenames="$(git diff-tree -r --no-commit-id --name-only HEAD "$1" | ( grep -P '^(xep-[0-9]{4}|inbox/[^/]+)\.xml$' || true))" +if [ -z "$filenames" ]; then + exit 0 +fi +mkdir -p rendered-changes/ +cp xmpp.css prettify.css rendered-changes/ +for filename in $filenames; do + built_filename="build/${filename/%.xml/.html}" + cp "$built_filename" rendered-changes/ +done diff --git a/tools/ci-prune-build.sh b/tools/ci-prune-build.sh new file mode 100644 index 00000000..fdc64b72 --- /dev/null +++ b/tools/ci-prune-build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -euo pipefail +outdir="build" +# clean out tex build logs etc. +find "$outdir" -type f '(' -iname "*.aux" -o -iname "*.log" -o -iname "*.toc" -o -iname "*.tex" -o -iname "*.tex.xml" -o -iname "*.out" ')' -print0 | xargs -0r -- rm + +find "$outdir" -type f '(' -iname "*.xml" -o -iname "*.html" -o -iname "*.pdf" ')' -print0 | while read -d $'\0' filename; do + if [ "$filename" = 'build/xmpp.pdf' ] || [ "$filename" = 'build/xmpp-text.pdf' ] || [ "$filename" = 'build/xeplist.xml' ]; then + continue + fi + + if [[ "$filename" =~ build/refs/reference.*.xml ]]; then + base_filename="$(echo "$filename" | sed -r 's#^build/refs/reference\.XSF\.XEP-([0-9]+)\.xml#xep-\1.xml#')" + else + base_filename="$(echo "$filename" | sed -r 's#^build/(.+)\.[^.]+$#\1.xml#')" + fi + + if [ ! -e "$base_filename" ]; then + printf 'deleting %q for which no source file (%q) exists\n' "$filename" "$base_filename" + rm "$filename" + fi +done diff --git a/tools/ci-restore-timestamps.py b/tools/ci-restore-timestamps.py new file mode 100644 index 00000000..2b5c7296 --- /dev/null +++ b/tools/ci-restore-timestamps.py @@ -0,0 +1,71 @@ +#!/usr/bin/python3 +import os +import pathlib +import subprocess +import time + + +def parse_timestamp_line(s: str): + author_ts, committer_ts = s.split(" ", 1) + return max(int(author_ts), int(committer_ts)) + + +def restore_commit_timestamps(basedir: pathlib.Path): + env = dict(os.environ) + env["LANG"] = "C.UTF-8" + # NOTE: the build image is still only on Python 3.4 because texml and stuff + # so we cannot use encoding= here and have to do decoding ourselves. + proc = subprocess.Popen( + [ + "git", "log", "--pretty=%at %ct", "--name-status", + ], + stdout=subprocess.PIPE, + env=env, + ) + + seen = set() + last_timestamp = None + for line in proc.stdout: + if not line: + continue + if not line.endswith(b"\n"): + raise ValueError("line not terminated") + line = line[:-1].decode("utf-8") + if not line: + continue + + try: + timestamp = parse_timestamp_line(line) + except ValueError: + pass + else: + last_timestamp = timestamp + continue + + _, filename = line.split("\t", 1) + if filename in seen: + continue + seen.add(filename) + filepath = basedir / filename + try: + os.utime(str(filepath), (last_timestamp, last_timestamp)) + except FileNotFoundError: + pass + + +def main(): + basedir = pathlib.Path.cwd() + + t0 = time.monotonic() + try: + restore_commit_timestamps(basedir) + finally: + t1 = time.monotonic() + print("timestamp restoration took {:.2f}s".format(t1-t0)) + + return 0 + + +if __name__ == "__main__": + import sys + sys.exit(main() or 0) diff --git a/tools/send-updates.py b/tools/send-updates.py index 680d1b9c..ae38e101 100755 --- a/tools/send-updates.py +++ b/tools/send-updates.py @@ -133,6 +133,10 @@ def dummy_info(number): } +def extract_version(info): + return info.get("last_revision", {}).get("version") + + def diff_infos(old, new): if old["status"] != new["status"]: if new["status"] == Status.PROTO: @@ -151,8 +155,8 @@ def diff_infos(old, new): old["last_call"] != new["last_call"]): return Action.LAST_CALL - old_version = old.get("last_revision", {}).get("version") - new_version = new.get("last_revision", {}).get("version") + old_version = extract_version(old) + new_version = extract_version(new) if old_version != new_version: return Action.UPDATE @@ -160,6 +164,32 @@ def diff_infos(old, new): return None +def decompose_version(s): + version_info = list(s.split(".")) + if len(version_info) < 3: + version_info.extend(['0'] * (3 - len(version_info))) + return version_info + + +def filter_bump_level(old_version, new_version, + include_editorial, include_non_editorial): + if old_version is None: + # treat as non-editorial + is_editorial = False + else: + old_version_d = decompose_version(old_version) + new_version_d = decompose_version(new_version) + # if the version number only differs in patch level or below, the change + # is editorial + is_editorial = old_version_d[:2] == new_version_d[:2] + + if is_editorial and not include_editorial: + return False + if not is_editorial and not include_non_editorial: + return False + return True + + def wraptext(text): return "\n".join( itertools.chain( @@ -262,10 +292,10 @@ def main(): ) parser.add_argument( "--no-proto", - dest="process_proto", + dest="include_protoxep", default=True, action="store_false", - help="Disable processing of ProtoXEPs.", + help="Do not announce ProtoXEPs", ) parser.add_argument( "-n", "--dry-run", @@ -274,6 +304,20 @@ def main(): default=False, help="Instead of sending emails, print them to stdout (implies -y)", ) + parser.add_argument( + "--no-editorial", + action="store_false", + default=True, + dest="include_editorial", + help="Do not announce editorial changes." + ) + parser.add_argument( + "--no-non-editorial", + action="store_false", + default=True, + dest="include_non_editorial", + help="Do not announce non-editorial changes." + ) parser.add_argument( "old", @@ -334,6 +378,13 @@ def main(): new_info = new_accepted[common_xep] action = diff_infos(old_info, new_info) + if action == Action.UPDATE and not filter_bump_level( + extract_version(old_info), + extract_version(new_info), + args.include_editorial, + args.include_non_editorial): + continue + if action is not None: updates.append((common_xep, action, new_info)) @@ -345,7 +396,7 @@ def main(): if action is not None: updates.append((added_xep, action, new_info)) - if args.process_proto: + if args.include_protoxep: for added_proto in added_protos: old_info = dummy_info('xxxx') new_info = new_proto[added_proto]