mirror of https://github.com/moparisthebest/xeps
Browse Source
This pipeline features the following: - Building of an nginx image with the XEPs as static files, in all formats. - Incremental builds on the main branch and incremental builds for MRs based on the last main build. - Automatic archiving of changed XEPs to the attic - Automatic announcement to the mailing listsmaster

7 changed files with 260 additions and 0 deletions
@ -0,0 +1,96 @@
@@ -0,0 +1,96 @@
|
||||
stages: |
||||
- build |
||||
- export |
||||
|
||||
"build@main": |
||||
image: registry.gitlab.com/xsf/docker-images/xep-buildspace/image:0.1.0 |
||||
stage: build |
||||
script: |
||||
- python3 tools/ci-restore-timestamps.py |
||||
- make html inbox-html inbox-xml pdf xeplist refs xml |
||||
- bash tools/ci-prune-build.sh |
||||
rules: |
||||
- if: '$CI_COMMIT_REF_NAME =~ /^main$/' |
||||
when: always |
||||
- when: never |
||||
cache: |
||||
key: build-cache |
||||
paths: |
||||
- build/ |
||||
artifacts: |
||||
paths: |
||||
- build/ |
||||
expire_in: '1 day' |
||||
resource_group: xep-build |
||||
|
||||
"pack@main": |
||||
image: docker:19.03.11 |
||||
stage: export |
||||
services: |
||||
- docker:19.03.11-dind |
||||
script: |
||||
- 'export IMAGE_REF="${CI_REGISTRY_IMAGE}/packed:main-$(date -Idate)-${CI_COMMIT_SHORT_SHA}"' |
||||
- 'export LATEST_REF="${CI_REGISTRY_IMAGE}/packed:main-latest"' |
||||
- 'docker build -t "$IMAGE_REF" -f pack-only.Dockerfile .' |
||||
- 'docker image tag "$IMAGE_REF" "$LATEST_REF"' |
||||
- 'docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY' |
||||
- 'docker push "$IMAGE_REF"' |
||||
- 'docker push "$LATEST_REF"' |
||||
rules: |
||||
- if: '$CI_COMMIT_REF_NAME =~ /^main$/' |
||||
when: on_success |
||||
- when: never |
||||
resource_group: xep-pack |
||||
|
||||
"attic@main": |
||||
image: python:3 |
||||
stage: export |
||||
script: |
||||
- bash -x ./tools/ci-archive.sh |
||||
cache: |
||||
paths: |
||||
- state/ |
||||
key: attic-state |
||||
rules: |
||||
- if: '$CI_COMMIT_REF_NAME =~ /^main$/' |
||||
when: on_success |
||||
- when: never |
||||
resource_group: xep-attic |
||||
|
||||
"announce@main": |
||||
image: python:3 |
||||
stage: export |
||||
script: |
||||
- bash -x ./tools/ci-announce.sh |
||||
cache: |
||||
paths: |
||||
- state/ |
||||
key: announce-state |
||||
rules: |
||||
- if: '$CI_COMMIT_REF_NAME =~ /^main$/' |
||||
when: on_success |
||||
- when: never |
||||
resource_group: xep-announce |
||||
|
||||
"build@mr": |
||||
image: registry.gitlab.com/xsf/docker-images/xep-buildspace-slim/image:0.1.1 |
||||
stage: build |
||||
script: |
||||
- python3 tools/ci-restore-timestamps.py |
||||
- make html inbox-html |
||||
- git fetch --depth=1 origin main |
||||
- bash tools/ci-changed-builds.sh origin/main |
||||
rules: |
||||
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"' |
||||
when: always |
||||
- when: never |
||||
cache: |
||||
key: build-cache |
||||
paths: |
||||
- build/ |
||||
policy: pull |
||||
artifacts: |
||||
expose_as: "Changed Documents" |
||||
paths: ["rendered-changes/"] |
||||
expire_in: '7 days' |
||||
|
@ -0,0 +1,5 @@
@@ -0,0 +1,5 @@
|
||||
FROM nginx:1-alpine |
||||
RUN mkdir /usr/share/nginx/html/extensions/ |
||||
COPY build/ /usr/share/nginx/html/extensions/ |
||||
RUN sed -ri '/root\s+\/usr\/share\/nginx\/html/s/^(.+)$/\1\nautoindex on;/' /etc/nginx/conf.d/default.conf |
||||
EXPOSE 80 |
@ -0,0 +1,19 @@
@@ -0,0 +1,19 @@
|
||||
#!/bin/bash |
||||
set -euo pipefail |
||||
state_dir=state |
||||
old_xeplist="$state_dir/old-xeplist.xml" |
||||
new_xeplist="build/xeplist.xml" |
||||
mkdir -p "$state_dir" |
||||
|
||||
function update_state() { |
||||
cp "$new_xeplist" "$old_xeplist" |
||||
} |
||||
|
||||
if [ ! -f "$old_xeplist" ]; then |
||||
printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2 |
||||
update_state |
||||
exit 0 |
||||
fi |
||||
|
||||
./tools/send-updates.py -y -c "$EMAIL_CFG" --no-editorial -- "$old_xeplist" "$new_xeplist" $EMAIL_RECIPIENTS |
||||
update_state |
@ -0,0 +1,34 @@
@@ -0,0 +1,34 @@
|
||||
#!/bin/bash |
||||
set -euo pipefail |
||||
state_dir=state |
||||
old_xeplist="$state_dir/old-xeplist.xml" |
||||
new_xeplist="build/xeplist.xml" |
||||
mkdir -p "$state_dir" |
||||
|
||||
function update_state() { |
||||
cp "$new_xeplist" "$old_xeplist" |
||||
} |
||||
|
||||
if [ ! -f "$old_xeplist" ]; then |
||||
printf '%q does not exist; assuming this is the first run!' "$old_xeplist" >&2 |
||||
update_state |
||||
exit 0 |
||||
fi |
||||
|
||||
chmod 0600 "$ATTIC_ID_RSA" |
||||
export GIT_SSH_COMMAND="ssh -i \"\$ATTIC_ID_RSA\" -o StrictHostKeyChecking=no" |
||||
git clone git@gitlab.com:xsf/xep-attic |
||||
python3 tools/archive.py -a xep-attic/content/ --no-build "$old_xeplist" "$new_xeplist" |
||||
pushd xep-attic |
||||
git add content |
||||
git update-index --refresh |
||||
if ! git diff-index --quiet HEAD --; then |
||||
git config user.name "$GIT_AUTHOR_NAME" |
||||
git config user.email "$GIT_AUTHOR_EMAIL" |
||||
git commit \ |
||||
-m "Automated XEP build ${CI_JOB_ID}" \ |
||||
-m "Job-URL: ${CI_JOB_URL}" |
||||
git push |
||||
fi |
||||
popd |
||||
update_state |
@ -0,0 +1,13 @@
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash |
||||
set -euo pipefail |
||||
IFS=$'\n' |
||||
filenames="$(git diff-tree -r --no-commit-id --name-only HEAD "$1" | ( grep -P '^(xep-[0-9]{4}|inbox/[^/]+)\.xml$' || true))" |
||||
if [ -z "$filenames" ]; then |
||||
exit 0 |
||||
fi |
||||
mkdir -p rendered-changes/ |
||||
cp xmpp.css prettify.css rendered-changes/ |
||||
for filename in $filenames; do |
||||
built_filename="build/${filename/%.xml/.html}" |
||||
cp "$built_filename" rendered-changes/ |
||||
done |
@ -0,0 +1,22 @@
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash |
||||
set -euo pipefail |
||||
outdir="build" |
||||
# clean out tex build logs etc. |
||||
find "$outdir" -type f '(' -iname "*.aux" -o -iname "*.log" -o -iname "*.toc" -o -iname "*.tex" -o -iname "*.tex.xml" -o -iname "*.out" ')' -print0 | xargs -0r -- rm |
||||
|
||||
find "$outdir" -type f '(' -iname "*.xml" -o -iname "*.html" -o -iname "*.pdf" ')' -print0 | while read -d $'\0' filename; do |
||||
if [ "$filename" = 'build/xmpp.pdf' ] || [ "$filename" = 'build/xmpp-text.pdf' ] || [ "$filename" = 'build/xeplist.xml' ]; then |
||||
continue |
||||
fi |
||||
|
||||
if [[ "$filename" =~ build/refs/reference.*.xml ]]; then |
||||
base_filename="$(echo "$filename" | sed -r 's#^build/refs/reference\.XSF\.XEP-([0-9]+)\.xml#xep-\1.xml#')" |
||||
else |
||||
base_filename="$(echo "$filename" | sed -r 's#^build/(.+)\.[^.]+$#\1.xml#')" |
||||
fi |
||||
|
||||
if [ ! -e "$base_filename" ]; then |
||||
printf 'deleting %q for which no source file (%q) exists\n' "$filename" "$base_filename" |
||||
rm "$filename" |
||||
fi |
||||
done |
@ -0,0 +1,71 @@
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/python3 |
||||
import os |
||||
import pathlib |
||||
import subprocess |
||||
import time |
||||
|
||||
|
||||
def parse_timestamp_line(s: str): |
||||
author_ts, committer_ts = s.split(" ", 1) |
||||
return max(int(author_ts), int(committer_ts)) |
||||
|
||||
|
||||
def restore_commit_timestamps(basedir: pathlib.Path): |
||||
env = dict(os.environ) |
||||
env["LANG"] = "C.UTF-8" |
||||
# NOTE: the build image is still only on Python 3.4 because texml and stuff |
||||
# so we cannot use encoding= here and have to do decoding ourselves. |
||||
proc = subprocess.Popen( |
||||
[ |
||||
"git", "log", "--pretty=%at %ct", "--name-status", |
||||
], |
||||
stdout=subprocess.PIPE, |
||||
env=env, |
||||
) |
||||
|
||||
seen = set() |
||||
last_timestamp = None |
||||
for line in proc.stdout: |
||||
if not line: |
||||
continue |
||||
if not line.endswith(b"\n"): |
||||
raise ValueError("line not terminated") |
||||
line = line[:-1].decode("utf-8") |
||||
if not line: |
||||
continue |
||||
|
||||
try: |
||||
timestamp = parse_timestamp_line(line) |
||||
except ValueError: |
||||
pass |
||||
else: |
||||
last_timestamp = timestamp |
||||
continue |
||||
|
||||
_, filename = line.split("\t", 1) |
||||
if filename in seen: |
||||
continue |
||||
seen.add(filename) |
||||
filepath = basedir / filename |
||||
try: |
||||
os.utime(str(filepath), (last_timestamp, last_timestamp)) |
||||
except FileNotFoundError: |
||||
pass |
||||
|
||||
|
||||
def main(): |
||||
basedir = pathlib.Path.cwd() |
||||
|
||||
t0 = time.monotonic() |
||||
try: |
||||
restore_commit_timestamps(basedir) |
||||
finally: |
||||
t1 = time.monotonic() |
||||
print("timestamp restoration took {:.2f}s".format(t1-t0)) |
||||
|
||||
return 0 |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
import sys |
||||
sys.exit(main() or 0) |
Loading…
Reference in new issue