From 585d719bd1725867647dd8cbbd42e69b4ed12b4a Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 13 Jul 2023 15:38:50 +0530 Subject: [PATCH] ci: Pick up the right branches when building a windows image Now we will pick up the right gstreamer branch + namespace when building an image, and also the right (matching, if any) cerbero branch + namespace. This solves the bootstrapping issue when doing an image update that requires coordination between gstreamer and cerbero. Part-of: --- .gitlab-ci.yml | 16 ++++++------ ci/docker/windows/Dockerfile | 8 +++++- ci/docker/windows/get_cerbero_clone_args.py | 27 +++++++++++++++++++++ ci/docker/windows/prepare_cerbero_env.sh | 8 ++++-- ci/docker/windows/prepare_gst_env.ps1 | 9 +++++-- 5 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 ci/docker/windows/get_cerbero_clone_args.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef0b3bf1c3..e83bdc4e4f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -180,14 +180,12 @@ windows amd64 docker: - "2022" script: # We need to pass an array and to resolve the env vars, so we can't use a variable: - - $DOCKER_BUILD_ARGS = @("--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH") - - - "& ci/docker/windows/container.ps1 $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_IMAGE $WINDOWS_UPSTREAM_IMAGE $DOCKERFILE" - - | - if (!($?)) { - echo "Failed to build the image" - Exit 1 - } + - $DOCKER_BUILD_ARGS = @(` + "--build-arg", "DEFAULT_BRANCH=$GST_UPSTREAM_BRANCH", ` + "--build-arg", "CI_PROJECT_NAMESPACE=$CI_PROJECT_NAMESPACE", ` + "--build-arg", "CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME" ` + ) + - ci/docker/windows/container.ps1 $CI_REGISTRY $CI_REGISTRY_USER $CI_REGISTRY_PASSWORD $WINDOWS_IMAGE $WINDOWS_UPSTREAM_IMAGE $DOCKERFILE # ---- Preparation ----- # @@ -503,7 +501,7 @@ build vs2022 arm64: --native-file ci/meson/vs2022-paths.ini --native-file ci/meson/vs2022-x64-native.ini -build msys2 : +build msys2: extends: '.build windows' timeout: '60min' script: diff --git a/ci/docker/windows/Dockerfile b/ci/docker/windows/Dockerfile index 3aa80dfb44..e4985dd48f 100644 --- a/ci/docker/windows/Dockerfile +++ b/ci/docker/windows/Dockerfile @@ -54,7 +54,7 @@ RUN C:\upgrade_msys2.ps1 RUN C:\msys64\msys2_shell.cmd -ucrt64 -defterm -here -no-start -use-full-path -lc 'pacman -S --noconfirm mingw-w64-ucrt-x86_64-toolchain ninja' -RUN python -m pip install meson==1.1.1 +RUN python -m pip install meson==1.1.1 python-gitlab RUN 'git config --global user.email "cirunner@gstreamer.freedesktop.org"; git config --global user.name "GStreamer CI system"' @@ -62,9 +62,15 @@ COPY install_rust.ps1 C:\ RUN C:\install_rust.ps1 ARG DEFAULT_BRANCH="main" +ARG CI_PROJECT_NAMESPACE +ARG CI_COMMIT_REF_NAME +ENV CI_PROJECT_NAMESPACE=$CI_PROJECT_NAMESPACE +ENV CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME COPY prepare_gst_env.ps1 C:\ RUN C:\prepare_gst_env.ps1 +ENV CI="true" +COPY get_cerbero_clone_args.py C:\ COPY prepare_cerbero_env.sh C:\ RUN C:\msys64\msys2_shell.cmd -ucrt64 -defterm -here -no-start -use-full-path -lc "C:/prepare_cerbero_env.sh" diff --git a/ci/docker/windows/get_cerbero_clone_args.py b/ci/docker/windows/get_cerbero_clone_args.py new file mode 100644 index 0000000000..4fdf8717cb --- /dev/null +++ b/ci/docker/windows/get_cerbero_clone_args.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 + +import os +import gitlab + + +server = 'https://gitlab.freedesktop.org' +gl = gitlab.Gitlab(server) +branch = os.environ.get('DEFAULT_BRANCH', 'main') +project = f'gstreamer/cerbero' +# We do not want to run on (often out of date) user upstream branch +if os.environ["CI_COMMIT_REF_NAME"] != os.environ['DEFAULT_BRANCH']: + try: + try_project = f'{os.environ["CI_PROJECT_NAMESPACE"]}/cerbero' + match_branch = os.environ["CI_COMMIT_REF_NAME"] + # Search for matching branches, return only if the branch name matches + # exactly + proj = gl.projects.get(try_project) + for b in proj.branches.list(search=match_branch, iterator=True): + if match_branch == b.name: + project = try_project + branch = b.name + break + except gitlab.exceptions.GitlabGetError: + pass + +print(f'-b {branch} {server}/{project}', end='') diff --git a/ci/docker/windows/prepare_cerbero_env.sh b/ci/docker/windows/prepare_cerbero_env.sh index fcf187b58e..2a54c4cb74 100644 --- a/ci/docker/windows/prepare_cerbero_env.sh +++ b/ci/docker/windows/prepare_cerbero_env.sh @@ -2,8 +2,12 @@ set -eux -cd C:/ -git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/cerbero.git C:/cerbero +# Clone `-b main gstreamer/cerbero` by default, but if running on a gstreamer +# branch in another namespace that has a corresponding cerbero branch by the +# same name, clone that instead. +clone_args="$(py -3 C:/get_cerbero_clone_args.py)" +echo "Cloning Cerbero using $clone_args" +git clone $clone_args C:/cerbero cd C:/cerbero echo 'local_sources="C:/cerbero/cerbero-sources"' > localconf.cbc diff --git a/ci/docker/windows/prepare_gst_env.ps1 b/ci/docker/windows/prepare_gst_env.ps1 index fd2f70840c..51f6926554 100644 --- a/ci/docker/windows/prepare_gst_env.ps1 +++ b/ci/docker/windows/prepare_gst_env.ps1 @@ -14,8 +14,13 @@ $secure_pw = ConvertTo-SecureString $plaintext_pw -AsPlainText -Force C:\msys64\ucrt64\bin\openssl.exe pkcs12 -export -nokeys -out $env:TEMP\certs.pfx -in $cert_pem -passout pass:$plaintext_pw Import-PfxCertificate -Password $secure_pw -CertStoreLocation Cert:\LocalMachine\Root -FilePath $env:TEMP\certs.pfx -Write-Host "Cloning GStreamer" -git clone -b $env:DEFAULT_BRANCH https://gitlab.freedesktop.org/gstreamer/gstreamer.git C:\gstreamer +$namespace = "gstreamer" +if ($env:CI_COMMIT_REF_NAME -ne $env:DEFAULT_BRANCH) { + $namespace = $env:CI_PROJECT_NAMESPACE +} +$url = "https://gitlab.freedesktop.org/$namespace/gstreamer.git" +Write-Host "Cloning GStreamer branch $env:CI_COMMIT_REF_NAME from $url" +git clone -b $env:CI_COMMIT_REF_NAME $url C:\gstreamer # download the subprojects to try and cache them Write-Host "Downloading subprojects"