From 63583a0bb73dca876bb59367f8b07763dfd9f529 Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Wed, 9 Jun 2021 15:41:44 -0400 Subject: [PATCH] CI: add jobs using Zuul It also includes a few changes to get the builds going: - Added autoconf to common dependencies - Added automake to common dependencies - Added libtool to common dependencies - Added libssl-dev to common dependencies Co-authored-by: Albin Vass Closes #7245 --- zuul.d/jobs.yaml | 542 ++++++++++++++++++++++++++++++++++ zuul.d/playbooks/.zuul.ignore | 0 zuul.d/playbooks/post.yaml | 12 + zuul.d/playbooks/pre.yaml | 70 +++++ zuul.d/playbooks/run.yaml | 12 + 5 files changed, 636 insertions(+) create mode 100644 zuul.d/jobs.yaml create mode 100644 zuul.d/playbooks/.zuul.ignore create mode 100644 zuul.d/playbooks/post.yaml create mode 100644 zuul.d/playbooks/pre.yaml create mode 100644 zuul.d/playbooks/run.yaml diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml new file mode 100644 index 000000000..77ec78b83 --- /dev/null +++ b/zuul.d/jobs.yaml @@ -0,0 +1,542 @@ +--- +- job: + name: curl-base + abstract: true + pre-run: zuul.d/playbooks/pre.yaml + run: zuul.d/playbooks/run.yaml + post-run: zuul.d/playbooks/post.yaml + nodeset: ubuntu-bionic + timeout: 3600 + vars: + curl_env: + LD_LIBRARY_PATH: /usr/local/lib + # NOTE(mnaser): Workaround to keep existing Travis scripts compatible + TRAVIS_OS_NAME: linux + +- job: + name: curl-normal-with-openssl-gssapi-libssh2-checksrc + parent: curl-base + vars: + curl_env: + T: normal + C: --with-openssl --with-gssapi --with-libssh2 + CHECKSRC: 1 + curl_apt_packages: + - krb5-user + - libssh2-1-dev + - libbrotli-dev + - libzstd-dev + +- job: + name: curl-normal-with-openssl-enable-ares + parent: curl-base + vars: + curl_env: + CC: gcc-8 + CXX: g++-8 + T: normal + C: >- + --with-openssl + --enable-ares + +- job: + name: curl-normal-with-openssl-disable-proxy + parent: curl-base + vars: + curl_env: + T: normal + C: >- + --with-openssl + --disable-proxy + +- job: + name: curl-normal-with-openssl-disable-verbose-notests + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: normal + C: >- + --with-openssl + --disable-verbose + CPPFLAGS: -Wno-variadic-macros + NOTESTS: 1 + +- job: + name: curl-novalgrind-boringssl-with-openssl + parent: curl-base + vars: + gimme_stable: true + curl_env: + CC: gcc-8 + CXX: g++-8 + T: novalgrind + BORINGSSL: "yes" + C: >- + --with-openssl={{ ansible_user_dir }}/boringssl + LD_LIBRARY_PATH: "{{ ansible_user_dir }}/boringssl/lib:/usr/local/lib" + +- job: + name: curl-novalgrind-boringssl-with-openssl-quiche + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: novalgrind + QUICHE: "yes" + C: >- + --with-openssl={{ ansible_user_dir }}/quiche/deps/boringssl/src + --with-quiche={{ ansible_user_dir }}/quiche/target/release + LD_LIBRARY_PATH: "{{ ansible_user_dir }}/quiche/target/release:/usr/local/lib" + +- job: + name: curl-debug-rustls + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + T: debug-rustls + RUSTLS_VERSION: v0.6.0 + C: >- + --with-rustls={{ ansible_user_dir }}/crust + +- job: + name: curl-debug-bearssl + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + T: debug-bearssl + BEARSSL: "yes" + C: >- + --with-bearssl + +- job: + name: curl-novalgrind-libressl + parent: curl-base + vars: + curl_env: + CC: gcc-8 + CXX: g++-8 + T: novalgrind + LIBRESSL: "yes" + C: >- + --with-openssl={{ ansible_user_dir }}/libressl + LD_LIBRARY_PATH: "{{ ansible_user_dir }}/libressl/lib:/usr/local/lib" + +- job: + name: curl-novalgrind-ngtcp2-with-openssl + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: novalgrind + NGTCP2: "yes" + C: >- + --with-openssl={{ ansible_user_dir }}/ngbuild + --with-ngtcp2={{ ansible_user_dir }}/ngbuild + --with-nghttp3={{ ansible_user_dir }}/ngbuild + NOTESTS: + +- job: + name: curl-novalgrind-ngtcp2-gnutls + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + - autogen + - automake + - autopoint + - bison + - gperf + - libgmp-dev + - libopts25-dev + - libp11-kit-dev + - libtasn1-6-dev + - nettle-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: novalgrind + NGTCP2: "yes" + GNUTLS: "yes" + C: >- + PKG_CONFIG_PATH={{ ansible_user_dir }}/ngbuild + --with-gnutls={{ ansible_user_dir }}/ngbuild + --with-ngtcp2={{ ansible_user_dir }}/ngbuild + --with-nghttp3={{ ansible_user_dir }}/ngbuild + NOTESTS: + +- job: + name: curl-debug-wolfssl + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: debug-wolfssl + WOLFSSL: "yes" + C: >- + --with-wolfssl + +- job: + name: curl-debug-openssl3 + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: debug + OPENSSL3: "yes" + C: >- + --with-openssl={{ ansible_user_dir }}/openssl3 + LD_LIBRARY_PATH: "{{ ansible_user_dir }}/openssl3/lib:/usr/local/lib" + TFLAGS: https ftps + +- job: + name: curl-debug-mesalink + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: debug-mesalink + MESALINK: "yes" + C: >- + --with-mesalink + +- job: + name: curl-debug-clang-with-openssl + parent: curl-base + vars: + curl_apt_packages: + - clang-9 + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: clang-9 + CXX: clang++-9 + T: debug + C: >- + --with-openssl + +- job: + name: curl-debug-clang-disable-alt-svc-with-openssl + parent: curl-base + vars: + curl_apt_packages: + - clang-9 + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: clang-9 + CXX: clang++-9 + T: debug + C: >- + --with-openssl + --disable-alt-svc + +- job: + name: curl-debug-clang-with-mbedtls + parent: curl-base + vars: + curl_apt_packages: + - clang-9 + - libpsl-dev + - libbrotli-dev + - libzstd-dev + - libmbedtls-dev + curl_env: + CC: clang-9 + CXX: clang++-9 + T: debug + C: >- + --with-mbedtls + +- job: + name: curl-debug-clang-with-gnutls + parent: curl-base + vars: + curl_apt_packages: + - clang-9 + - libpsl-dev + - libbrotli-dev + - libzstd-dev + - libgnutls28-dev + curl_env: + CC: clang-9 + CXX: clang++-9 + T: debug + C: >- + --with-gnutls + +- job: + name: curl-debug-clang-with-nss + parent: curl-base + vars: + curl_apt_packages: + - clang-9 + - libpsl-dev + - libbrotli-dev + - libzstd-dev + - libnss3-dev + curl_env: + CC: clang-9 + CXX: clang++-9 + T: debug + C: >- + --with-nss + CPPFLAGS: -isystem /usr/include/nss + NOTESTS: 1 + +- job: + name: curl-iconv-with-openssl + parent: curl-base + vars: + curl_env: + CC: gcc-8 + CXX: g++-8 + T: iconv + C: >- + --with-openssl + +- job: + name: curl-cmake-boringssl-quiche + parent: curl-base + vars: + gimme_stable: true + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + BORINGSSL: "yes" + QUICHE: "yes" + CC: gcc-8 + CXX: g++-8 + T: cmake + C: >- + -GNinja + -DUSE_QUICHE=1 + -DOPENSSL_ROOT_DIR={{ ansible_user_dir }}/boringssl + -DCURL_BROTLI=1 + -DCURL_ZSTD=1 + TFLAGS: https ftps + PKG_CONFIG_PATH: "{{ ansible_user_dir }}/quiche/target/release" + +- job: + name: curl-cmake-ngtcp2 + parent: curl-base + vars: + curl_apt_packages: + - clang-9 + - libpsl-dev + - libbrotli-dev + - libzstd-dev + - libnss3-dev + curl_env: + NGTCP2: "yes" + CC: clang-9 + CXX: clang++-9 + T: cmake + C: >- + -GNinja + -DUSE_NGTCP2=ON + -DCURL_BROTLI=1 + -DCURL_ZSTD=1 + PKG_CONFIG_PATH: "{{ ansible_user_dir }}/ngbuild/lib/pkgconfig" + +- job: + name: curl-torture + parent: curl-base + vars: + curl_apt_packages: + - lcov + - libpsl-dev + - libssl-dev + - libbrotli-dev + - libzstd-dev + - libssh2-1-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: torture + +- job: + name: curl-events + parent: curl-base + vars: + curl_apt_packages: + - lcov + - libpsl-dev + - libssl-dev + - libbrotli-dev + - libzstd-dev + - libssh2-1-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: events + +- job: + name: curl-distcheck + parent: curl-base + vars: + curl_apt_packages: + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: gcc-8 + CXX: g++-8 + T: distcheck + +- job: + name: curl-fuzzer + parent: curl-base + vars: + curl_apt_packages: + - clang + - clang-9 + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: clang-9 + CXX: clang++-9 + T: fuzzer + +- job: + name: curl-tidy + parent: curl-base + vars: + curl_apt_packages: + - clang + - clang-tidy + - clang-9 + - clang-tidy-9 + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: clang-9 + CXX: clang++-9 + T: tidy + C: --with-openssl + +- job: + name: curl-scan-build + parent: curl-base + vars: + curl_apt_packages: + - clang-tools-10 + - clang-9 + - libssl-dev + - libssh2-1-dev + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: clang-9 + CXX: clang++-9 + T: scan-build + C: >- + --with-openssl + --with-libssh2 + +- job: + name: curl-debug-clang-with-openssl-dl-ubsan + parent: curl-base + vars: + curl_apt_packages: + - clang-9 + - libpsl-dev + - libbrotli-dev + - libzstd-dev + curl_env: + CC: clang-9 + CXX: clang++-9 + T: debug + CFLAGS: >- + -fsanitize=address,undefined,signed-integer-overflow + -fno-sanitize-recover=undefined,integer + -Wformat + -Werror=format-security + -Werror=array-bounds + -g + LDFLAGS: >- + -fsanitize=address,undefined + -fno-sanitize-recover=undefined,integer + LIBS: -ldl -lubsan + TFLAGS: -n + C: --with-openssl + +- project: + check: + jobs: + - curl-normal-with-openssl-gssapi-libssh2-checksrc + - curl-normal-with-openssl-enable-ares + - curl-normal-with-openssl-disable-proxy + - curl-normal-with-openssl-disable-verbose-notests + - curl-novalgrind-boringssl-with-openssl + - curl-debug-rustls + - curl-debug-bearssl + - curl-novalgrind-libressl + - curl-novalgrind-ngtcp2-with-openssl + - curl-novalgrind-ngtcp2-gnutls + - curl-debug-wolfssl + - curl-debug-wolfssl + - curl-debug-mesalink + - curl-debug-clang-with-openssl + - curl-debug-clang-disable-alt-svc-with-openssl + - curl-debug-clang-with-mbedtls + - curl-debug-clang-with-gnutls + - curl-debug-clang-with-nss + - curl-iconv-with-openssl + - curl-cmake-boringssl-quiche + - curl-cmake-ngtcp2 + - curl-torture + - curl-events + - curl-distcheck + - curl-fuzzer + - curl-tidy + - curl-scan-build + - curl-debug-clang-with-openssl-dl-ubsan +... diff --git a/zuul.d/playbooks/.zuul.ignore b/zuul.d/playbooks/.zuul.ignore new file mode 100644 index 000000000..e69de29bb diff --git a/zuul.d/playbooks/post.yaml b/zuul.d/playbooks/post.yaml new file mode 100644 index 000000000..2df7a778f --- /dev/null +++ b/zuul.d/playbooks/post.yaml @@ -0,0 +1,12 @@ +--- +- hosts: all + tasks: + - name: Pull down logs to executor + ignore_errors: True + synchronize: + src: "{{ zuul.project.src_dir }}/config.log" + dest: "{{ zuul.executor.log_root }}/config.log" + mode: pull + owner: false + group: false +... diff --git a/zuul.d/playbooks/pre.yaml b/zuul.d/playbooks/pre.yaml new file mode 100644 index 000000000..10334736f --- /dev/null +++ b/zuul.d/playbooks/pre.yaml @@ -0,0 +1,70 @@ +--- +- hosts: all + tasks: + + - name: Install latest stable release of go + when: gimme_stable|default(false) + block: + - name: Find latest stable version + register: go_stable + uri: + url: https://golang.org/VERSION?m=text + return_content: true + - name: Install Go + include_role: + name: ensure-go + vars: + go_version: "{{ go_stable.content | regex_replace('^go', '') }}" + + - name: Symlink /usr/local/go/bin/go to /usr/bin/go + become: true + file: + src: /usr/local/go/bin/go + dest: /usr/bin/go + state: link + + - name: Install common dependencies + become: true + apt: + update_cache: true + pkg: + - autoconf + - automake + - cmake + - valgrind + - libev-dev + - libc-ares-dev + - libssl-dev + - libtool + - g++ + - g++-8 + - stunnel4 + - libidn2-dev + - gnutls-bin + - python-impacket + - ninja-build + - libgsasl7-dev + - libnghttp2-dev + + - name: Install job-specific packages + when: curl_apt_packages is defined + become: true + apt: + pkg: "{{ curl_apt_packages }}" + + - name: Symlink /usr/bin/scan-build-10 to /usr/bin/scan-build + when: + - curl_apt_packages is defined + - '"clang-tools-10" in curl_apt_packages' + become: true + file: + src: /usr/bin/scan-build-10 + dest: /usr/bin/scan-build + state: link + + - name: Run before script + shell: "./scripts/travis/before_script.sh" + args: + chdir: "{{ zuul.project.src_dir }}" + environment: "{{ curl_env }}" +... diff --git a/zuul.d/playbooks/run.yaml b/zuul.d/playbooks/run.yaml new file mode 100644 index 000000000..862fbaa51 --- /dev/null +++ b/zuul.d/playbooks/run.yaml @@ -0,0 +1,12 @@ +--- +- hosts: all + tasks: + - name: Print environment variables + debug: + var: curl_env + + - name: Run tests + environment: "{{ curl_env }}" + shell: "./scripts/travis/script.sh" + args: + chdir: "{{ zuul.project.src_dir }}" \ No newline at end of file