Philipp Hörist pushed to branch master at gajim / gajim
Commits: e09c2c82 by lovetox at 2022-03-26T19:33:58+01:00 Refactor CI Pipelines
- Move tasks to python scripts - Rework appveyor build - Add deploy jobs
- - - - -
5 changed files:
- .gitlab-ci.yml - + scripts/ci/deploy.py - + scripts/ci/sdist.py - appveyor.yml → win/ci/appveyor.yml - + win/ci/appveyor_build.py
Changes:
===================================== .gitlab-ci.yml ===================================== @@ -1,14 +1,19 @@ image: ci-gajim:master
-include: -- project: 'gajim/ci' - ref: master - file: '.gajim-ci.yml' +workflow: + rules: + - if: $GAJIM_NIGHTLY_BUILD + - if: $CI_PIPELINE_SOURCE == "push" + +stages: + - test + - build + - deploy
pyright: stage: test script: - - pip3 install git+https://dev.gajim.org/gajim/python-nbxmpp.git@$NBXMPP_BRANCH + - pip3 install git+https://dev.gajim.org/gajim/python-nbxmpp.git - pip3 install git+https://github.com/pygobject/pygobject-stubs.git - npm install pyright - python3 -V @@ -16,8 +21,77 @@ pyright: - $(npm bin)/pyright --version - $(npm bin)/pyright interruptible: true - allow_failure: false
-variables: - NBXMPP_BRANCH: master - PLUGINS_BRANCH: master +code-quality: + stage: test + script: + - python3 -V + - pip3 install -I git+https://dev.gajim.org/gajim/python-nbxmpp.git + - scripts/ci/pylint.sh --jobs=2 gajim + - coverage run --source=gajim -m unittest discover -s test -v + - coverage report -mi + - coverage xml -i + - codespell -I codespell.conf --skip="*__pycache__*,gajim/data/icons,gajim/data/sounds,gajim/data/emoticons" gajim + - python3 setup.py build + - appstream-util validate build/data/org.gajim.Gajim.appdata.xml + coverage: "/TOTAL.+ ([0-9]{1,3}%)/" + artifacts: + reports: + cobertura: coverage.xml + interruptible: true + +build-linux: + stage: build + rules: + - if: '$GAJIM_NIGHTLY_BUILD' + - if: '$CI_COMMIT_TAG' + when: manual + script: + - python3 ./scripts/ci/sdist.py + + artifacts: + name: "gajim-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA" + expire_in: 1 week + paths: + - dist/gajim-*.tar.gz + +build-windows: + stage: build + rules: + - if: '$GAJIM_NIGHTLY_BUILD' + - if: '$CI_COMMIT_TAG' + script: + - python3 ./win/ci/appveyor_build.py + + artifacts: + expire_in: 1 day + paths: + - build/Gajim-*.exe + +deploy-windows: + stage: deploy + needs: ['build-windows'] + rules: + - if: '$GAJIM_NIGHTLY_BUILD' + - if: '$CI_COMMIT_TAG' + when: manual + variables: + DEPLOY_TYPE: "windows" + script: + - python3 ./scripts/ci/deploy.py build + +deploy-flatpak: + stage: deploy + needs: [] + rules: + - if: '$CI_COMMIT_TAG' + when: manual + script: + - git clone https://github.com/flathub/org.gajim.Gajim.git + - cd org.gajim.Gajim + - git config user.email "ci@gajim.org" + - git config user.name "CI Deploy" + - mv ../flatpak/org.gajim.Gajim.yaml org.gajim.Gajim.yaml + - git diff + - git add org.gajim.Gajim.yaml + - git commit -m "$CI_COMMIT_TAG"
===================================== scripts/ci/deploy.py ===================================== @@ -0,0 +1,97 @@ +import os +import sys +from ftplib import FTP_TLS, ftpcp +from pathlib import Path +import functools +from typing import Any + +from rich.console import Console + +FTP_URL = 'panoramix.gajim.org' +FTP_USER = os.environ['FTP_USER'] +FTP_PASS = os.environ['FTP_PASS'] + +WINDOWS_NIGHTLY_FOLDER = 'win_snap' + +console = Console() + + +def ftp_connection(func: Any) -> Any: + @functools.wraps(func) + def func_wrapper(filedir: Path) -> None: + ftp = FTP_TLS(FTP_URL, FTP_USER, FTP_PASS) + console.print('Successfully connected to', FTP_URL) + func(ftp, filedir) + ftp.quit() + console.print('Quit') + return + return func_wrapper + + +def get_release_folder_from_tag(tag: str) -> str: + numbers = tag.split('.') + return '.'.join(numbers[:2]) + + +def get_gajim_tag() -> str: + tag = os.environ.get('CI_COMMIT_TAG') + if tag is None: + exit('No tag found') + return tag.removesuffix('gajim-') + + +def get_dir_list(ftp: FTP_TLS) -> list[str]: + return [x[0] for x in ftp.mlsd()] + + +def ensure_folder_exists(ftp: FTP_TLS, dirname: str) -> None: + dir_list = get_dir_list(ftp) + if dirname not in dir_list: + ftp.mkd(dirname) + + +def upload_all_from_dir(ftp: FTP_TLS, dir: Path) -> None: + for file_path in dir.iterdir(): + console.print('Upload file', file_path.name) + with open(file_path, 'rb') as f: + ftp.storbinary('STOR ' + file_path.name, f) + + +def get_deploy_method() -> str: + deploy_type = os.environ['DEPLOY_TYPE'] + is_nightly = bool(os.environ.get('GAJIM_NIGHTLY_BUILD')) + if is_nightly: + return f'deploy_{deploy_type}_nightly' + return f'deploy_{deploy_type}_release' + + +@ftp_connection +def deploy_windows_nightly(ftp: FTP_TLS, filedir: Path) -> None: + ftp.cwd(WINDOWS_NIGHTLY_FOLDER) + upload_all_from_dir(ftp, filedir) + + +@ftp_connection +def deploy_windows_release(ftp: FTP_TLS, filedir: Path) -> None: + tag = get_gajim_tag() + folder = get_release_folder_from_tag(tag) + ensure_folder_exists(ftp, folder) + ftp.cwd(folder) + upload_all_from_dir(ftp, filedir) + + +@ftp_connection +def deploy_linux_nightly(): + raise NotImplementedError + + +@ftp_connection +def deploy_linux_release(): + raise NotImplementedError + + +if __name__ == '__main__': + filedir = Path(sys.argv[1]) + current_module = sys.modules[__name__] + method = getattr(current_module, get_deploy_method()) + method(filedir)
===================================== scripts/ci/sdist.py ===================================== @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import io +import zipfile +import subprocess +import shutil +from pathlib import Path +import requests + +PLUGINS = [ + 'plugin_installer', +] + +PLUGINS_BASE_URL = 'https://ftp.gajim.org' +PLUGINS_FOLDER = Path('./gajim/data/plugins') + + +def get_plugins_url(plugin): + return f'{PLUGINS_BASE_URL}/plugins_master_zip/{plugin}.zip' + + +def extraxt_zip(zip_bytes, path): + print('Extract to', path) + with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zip_file: + zip_file.extractall(path) + + +def download_plugins(): + PLUGINS_FOLDER.mkdir(parents=True) + for plugin in PLUGINS: + url = get_plugins_url(plugin) + print('Download', url) + req = requests.get(url) + req.raise_for_status() + extraxt_zip(req.content, PLUGINS_FOLDER) + + +def setup(): + print('Setup') + subprocess.call(['python3', 'setup.py', 'sdist']) + + +def cleanup(): + print('Cleanup') + shutil.rmtree(PLUGINS_FOLDER) + + +download_plugins() +setup() +cleanup()
===================================== appveyor.yml → win/ci/appveyor.yml ===================================== @@ -1,59 +1,57 @@ -image: Visual Studio 2019 - -environment: - matrix: - - MSYSTEM: MINGW64 - MSYS_ARCH: "x86_64" - ARCH: "64bit" - - - MSYSTEM: MINGW32 - MSYS_ARCH: "i686" - ARCH: "32bit" - -branches: - only: - - master - -clone_depth: 1 - -# init: -# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - -install: - - set PATH=C:\msys64\usr\bin;%PATH% - - bash -lc "pacman --needed --noconfirm -Syu" - # This is needed because without killing all processes -Su will fail - - ps: Get-Process | Where-Object {$_.path -like 'C:\msys64*'} | Stop-Process - - bash -lc "pacman -Sydd --noconfirm filesystem" - - bash -lc "pacman --needed --noconfirm -Su" - -build_script: - - - ps: | - $env:TIME_STRING=(get-date -UFormat "%Y-%m-%d").ToString() - $env:BUILDROOT="C:\msys64\home\appveyor\gajim\win_build_root" - - function bash($command) { - Write-Host $command -NoNewline - C:\msys64\usr\bin\sh.exe --login -c $command - } - - bash "git clone C:/projects/gajim C:/msys64/home/appveyor/gajim" - bash "C:/msys64/home/appveyor/gajim/win/build.sh $($env:MSYS_ARCH)" - Push-AppveyorArtifact "$($env:BUILDROOT)/Gajim.exe" -FileName "Gajim-Master-$($env:ARCH)-$($env:TIME_STRING).exe" - Push-AppveyorArtifact "$($env:BUILDROOT)/Gajim-Portable.exe" -FileName "Gajim-Portable-Master-$($env:ARCH)-$($env:TIME_STRING).exe" - -# on_finish: -# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - -deploy: - # FTP deployment provider settings - - provider: FTP - protocol: ftp - host: panoramix.gajim.org - username: - secure: SNcOJDhUtBjfddbKXudE2w== - password: - secure: tQkYbcUb6nChCp0cdqo4CA== - folder: win_snap - debug: true +version: 1.1.{build} + +image: Visual Studio 2019 + +environment: + matrix: + - MSYSTEM: MINGW64 + MSYS_ARCH: "x86_64" + ARCH: "64bit" + + - MSYSTEM: MINGW32 + MSYS_ARCH: "i686" + ARCH: "32bit" + +branches: + only: + - master + +clone_depth: 1 + +# init: +# - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) + +install: + - set PATH=C:\msys64\usr\bin;%PATH% + - bash -lc "pacman --needed --noconfirm -Syu" + # This is needed because without killing all processes -Su will fail + - ps: Get-Process | Where-Object {$_.path -like 'C:\msys64*'} | Stop-Process + - bash -lc "pacman -Sydd --noconfirm filesystem" + - bash -lc "pacman --needed --noconfirm -Su" + +build_script: + + - ps: | + $filename = "Gajim-$($env:GAJIM_VERSION)-$($env:ARCH)" + $filename_portable = "Gajim-$($env:GAJIM_VERSION)-Portable-$($env:ARCH)" + + if ($env:GAJIM_VERSION -eq "Nightly") { + $time_string=(get-date -UFormat "%Y-%m-%d").ToString() + $filename = $filename + "-" + $time_string + $filename_portable = $filename_portable + "-" + $time_string + } + + $buildroot="C:\msys64\home\appveyor\gajim\win_build_root" + + function bash($command) { + Write-Host $command -NoNewline + C:\msys64\usr\bin\sh.exe --login -c $command + } + + bash "git clone C:/projects/gajim C:/msys64/home/appveyor/gajim" + bash "C:/msys64/home/appveyor/gajim/win/build.sh $($env:MSYS_ARCH)" + Push-AppveyorArtifact "$($buildroot)/Gajim.exe" -FileName "$($filename).exe" + Push-AppveyorArtifact "$($buildroot)/Gajim-Portable.exe" -FileName "$($filename_portable).exe" + +# on_finish: +# - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
===================================== win/ci/appveyor_build.py ===================================== @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +from typing import Iterator + +import os +import requests +import time +from pathlib import Path + +from rich.console import Console + +ACCOUNT = 'lovetox' +PROJECT_SLUG = 'gajim' +BRANCH = 'master' +BASE_URL = 'https://ci.appveyor.com/api' +API_KEY = os.environ['APPVEYOR_API_KEY'] +HEADERS = {'Authorization': f'Bearer {API_KEY}'} +RETRY_TIMEOUT = 2 * 60 +INITIAL_START_DELAY = 20 * 60 + +SETTINGS_API_URL = f'{BASE_URL}/projects/{ACCOUNT}/{PROJECT_SLUG}/settings/yaml' +BUILDS_API_URL = f'{BASE_URL}/builds' +PROJECT_API_URL = f'{BASE_URL}/projects/{ACCOUNT}/{PROJECT_SLUG}' + + +console = Console() + + +def get_gajim_version() -> str: + if os.environ.get('GAJIM_NIGHTLY_BUILD') is not None: + return 'Nightly' + + tag = os.environ.get('CI_COMMIT_TAG') + if tag is None: + exit('No tag found') + return tag.removesuffix('gajim-') + + +def push_yaml_to_project() -> None: + console.print('Push settings ...') + with open('./win/ci/appveyor.yml', 'r') as file: + yaml = file.read() + + req = requests.put(SETTINGS_API_URL, data=yaml, headers=HEADERS) + req.raise_for_status() + + +def start_build() -> str: + console.print('Start build ...') + payload = { + 'accountName': ACCOUNT, + 'projectSlug': PROJECT_SLUG, + 'branch': BRANCH, + 'environmentVariables': { + 'GAJIM_VERSION': get_gajim_version(), + } + } + req = requests.post(BUILDS_API_URL, headers=HEADERS, json=payload) + req.raise_for_status() + response = req.json() + return response['buildId'] + + +def is_build_finished(build: dict[str, str]) -> bool: + if build['status'] in ('failed', 'cancelled'): + exit('Found failed job') + + return build['status'] == 'success' + + +def get_artifacts(build_id: str) -> None: + time.sleep(INITIAL_START_DELAY) + while True: + time.sleep(RETRY_TIMEOUT) + + console.print('Check build status ...') + req = requests.get(PROJECT_API_URL, headers=HEADERS) + req.raise_for_status() + response = req.json() + build = response['build'] + if build_id != build['buildId']: + exit('Unable to find buildid: %s' % build_id) + + if is_build_finished(build): + break + + console.print('Build status:', build['status']) + + for job in build['jobs']: + download_job_artifacts(job['jobId']) + + console.print('All artifacts downloaded!') + + +def download_job_artifacts(job_id: str) -> None: + artifacts_api_url = f'{BASE_URL}/buildjobs/{job_id}/artifacts' + req = requests.get(artifacts_api_url, headers=HEADERS) + req.raise_for_status() + response = req.json() + + build_folder = Path.cwd() / 'build' + + for artifact in response: + filename = artifact['fileName'] + console.print('Download', filename, '...') + file_url = f'{artifacts_api_url}/{filename}' + req = requests.get(file_url, headers=HEADERS) + req.raise_for_status() + with open(build_folder / filename, 'wb') as file: + file.write(req.content) + + +if __name__ == '__main__': + push_yaml_to_project() + build_id = start_build() + get_artifacts(build_id)
View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/e09c2c82366a286942b2b023d7edcc04a...