Philipp Hörist pushed to branch port-pyproject at gajim / gajim

Commits: 81953846 by Philipp Hörist at 2023-01-20T14:32:12+01:00 Update ci

- - - - - ae5eba8b by Philipp Hörist at 2023-01-20T14:32:12+01:00 Update manifest.in

- - - - - c08e6423 by Philipp Hörist at 2023-01-20T14:43:28+01:00 ci: Update debian control file

- - - - - 5d35819c by Philipp Hörist at 2023-01-21T11:29:15+01:00 ci: Use release-helper for build and deploy

- - - - - d2851368 by Philipp Hörist at 2023-01-21T11:29:57+01:00 ci: Remove obsolete scripts

- - - - -

6 changed files:

- − .ci/appveyor_build.py - − .ci/debian_build.py - − .ci/deploy.py - .gitlab-ci.yml - MANIFEST.in - debian/control

Changes:

===================================== .ci/appveyor_build.py deleted ===================================== @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 - -from typing import Any - -import json -import logging -import os -import sys -import time -from pathlib import Path - -import requests - -ACCOUNT = 'lovetox' -PROJECT_SLUG = 'gajim' -BRANCH = 'master' -BASE_URL = 'https://ci.appveyor.com/api' -API_KEY = os.environ['APPVEYOR_API_KEY'] -HEADERS = {'Authorization': f'Bearer {API_KEY}'} -RETRY_TIMEOUT = 2 * 60 -INITIAL_START_DELAY = 20 * 60 - -SETTINGS_API_URL = f'{BASE_URL}/projects/{ACCOUNT}/{PROJECT_SLUG}/settings/yaml' -BUILDS_API_URL = f'{BASE_URL}/builds' -PROJECT_API_URL = f'{BASE_URL}/projects/{ACCOUNT}/{PROJECT_SLUG}' - - -logging.basicConfig(level='INFO', format='%(levelname)s: %(message)s') -log = logging.getLogger() - - -def get_gajim_version() -> str: - if os.environ.get('GAJIM_NIGHTLY_BUILD') is not None: - return 'Nightly' - - tag = os.environ.get('CI_COMMIT_TAG') - if tag is None: - sys.exit('No tag found') - return tag - - -def push_yaml_to_project() -> None: - log.info('Push settings ...') - with open('.ci/appveyor.yml') as file: - yaml = file.read() - - req = requests.put(SETTINGS_API_URL, data=yaml, headers=HEADERS) - req.raise_for_status() - - -def start_build() -> str: - log.info('Start build ...') - payload = { - 'accountName': ACCOUNT, - 'projectSlug': PROJECT_SLUG, - 'branch': BRANCH, - 'commitId': os.environ['CI_COMMIT_SHA'], - 'environmentVariables': { - 'GAJIM_VERSION': get_gajim_version(), - } - } - req = requests.post(BUILDS_API_URL, headers=HEADERS, json=payload) - req.raise_for_status() - response = req.json() - return response['buildId'] - - -def is_build_finished(build: dict[str, str]) -> bool: - if build['status'] in ('failed', 'cancelled'): - sys.exit('Found failed job') - - return build['status'] == 'success' - - -def check_for_response(build_id: str) -> None: - time.sleep(INITIAL_START_DELAY) - while True: - time.sleep(RETRY_TIMEOUT) - - log.info('Check build status ...') - req = requests.get(PROJECT_API_URL, headers=HEADERS) - req.raise_for_status() - response = req.json() - build = response['build'] - if build_id != build['buildId']: - sys.exit('Unable to find buildid: %s' % build_id) - - if is_build_finished(build): - break - - log.info('Build status: %s', build['status']) - - build_folder = Path.cwd() / 'build' - build_folder.mkdir() - - for job in build['jobs']: - response = get_job_response(job['jobId']) - result = build_folder / f'{job["jobId"]}.json' - result.write_text(json.dumps(response)) - log.info('Write job response: %s', result) - - -def get_job_response(job_id: str) -> list[dict[str, Any]]: - artifacts_api_url = f'{BASE_URL}/buildjobs/{job_id}/artifacts' - req = requests.get(artifacts_api_url, headers=HEADERS) - req.raise_for_status() - response = req.json() - - for artifact in response: - file_url = f'{artifacts_api_url}/{artifact["fileName"]}' - artifact['fileUrl'] = file_url - - return response - - -if __name__ == '__main__': - push_yaml_to_project() - build_id = start_build() - check_for_response(build_id)

===================================== .ci/debian_build.py deleted ===================================== @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import argparse -import dataclasses -import logging -import shutil -import subprocess -from datetime import datetime -from datetime import timezone -from pathlib import Path - -LOG_FORMAT = '%(asctime)s %(levelname)s %(message)s' -logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) -log = logging.getLogger() - -ROOT_DIR = Path(__file__).resolve().parent.parent -BUILD_DIR = ROOT_DIR / 'debian_build' - -DATE = datetime.now().strftime('%Y%m%d') -DATE_TIME = datetime.now(tz=timezone.utc).strftime('%a, %d %b %Y %T %z') - - -@dataclasses.dataclass -class ReleaseContext: - app: str - pkg_name: str - rev: str - release_name: str - release_dir: Path - tarball: Path - - @classmethod - def from_tarball(cls, path: str, prefix: str, rev: str) -> ReleaseContext: - tarball = Path(path) - app = tarball.name.split('-', maxsplit=1)[0] - pkg_name = f'{prefix}{app}-nightly' - release_name = f'{pkg_name}_{DATE}' - release_dir = BUILD_DIR / release_name - return cls(app=app, - pkg_name=pkg_name, - rev=rev, - release_name=release_name, - release_dir=release_dir, - tarball=tarball) - - -def clean_build_dir() -> None: - log.info('Cleanup build directory') - if BUILD_DIR.exists(): - shutil.rmtree(BUILD_DIR) - BUILD_DIR.mkdir() - - -def prepare_package_dir(context: ReleaseContext) -> None: - log.info('Extract tarball') - tarball = Path(shutil.copy(context.tarball, BUILD_DIR)) - tarball = tarball.rename(BUILD_DIR / f'{context.release_name}.orig.tar.gz') - shutil.unpack_archive(tarball, BUILD_DIR) - - log.info('Rename dir to: %s', context.release_name) - folder = list(BUILD_DIR.glob(f'{context.app}-?.?.?'))[0] - folder.rename(context.release_dir) - - log.info('Copy debian folder into release directory') - shutil.copytree(ROOT_DIR / 'debian', context.release_dir / 'debian') - - -def prepare_changelog(context: ReleaseContext) -> None: - log.info('Prepare Changelog') - changelog = context.release_dir / 'debian' / 'changelog' - content = changelog.read_text() - content = content.replace('{DATE}', f'{DATE}-{context.rev}') - content = content.replace('{DATE_TIME}', DATE_TIME) - changelog.write_text(content) - - -def build(context: ReleaseContext) -> None: - log.info('Start package build') - subprocess.run( - [ - 'dpkg-buildpackage', - '--no-sign' - ], - cwd=context.release_dir, - check=True) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Build debian package') - parser.add_argument('tarball', help='Path to tarball e.g. app.tar.gz') - parser.add_argument('rev', help='The package revision e.g. 1') - parser.add_argument('--pkgprefix', default='', required=False, - help='Prefix for the package name e.g. python3-') - args = parser.parse_args() - - context = ReleaseContext.from_tarball(args.tarball, - args.pkgprefix, - args.rev) - - clean_build_dir() - prepare_package_dir(context) - prepare_changelog(context) - build(context)

===================================== .ci/deploy.py deleted ===================================== @@ -1,165 +0,0 @@ -#!/usr/bin/env python3 - -from typing import Any -from typing import cast -from typing import Optional - -import functools -import json -import logging -import os -import sys -from datetime import date -from ftplib import error_perm -from ftplib import FTP_TLS -from pathlib import Path - -import requests - -FTP_URL = 'panoramix.gajim.org' -FTP_USER = os.environ['FTP_USER'] -FTP_PASS = os.environ['FTP_PASS'] - -API_KEY = os.environ['APPVEYOR_API_KEY'] -HEADERS = {'Authorization': f'Bearer {API_KEY}'} - -WINDOWS_NIGHTLY_FOLDER = 'downloads/snap/win' -LINUX_NIGHTLY_FOLDER = 'downloads/snap' - -RELEASE_FOLDER_BASE = 'downloads' - - -logging.basicConfig(level='INFO', format='%(levelname)s: %(message)s') -log = logging.getLogger() - - -def ftp_connection(func: Any) -> Any: - @functools.wraps(func) - def func_wrapper(filedir: Path) -> None: - ftp = FTP_TLS(FTP_URL, FTP_USER, FTP_PASS) - log.info('Successfully connected to %s', FTP_URL) - func(ftp, filedir) - ftp.quit() - log.info('Quit') - return - return func_wrapper - - -def get_release_folder_from_tag(tag: str) -> str: - numbers = tag.split('.') - return '.'.join(numbers[:2]) - - -def get_gajim_tag() -> str: - tag = os.environ.get('CI_COMMIT_TAG') - if tag is None: - sys.exit('No tag found') - return tag - - -def find_linux_tarball(filedir: Path) -> Path: - files = list(filedir.glob('gajim-*.tar.gz')) - if len(files) != 1: - sys.exit('Unknown files found') - return files[0] - - -def get_dir_list(ftp: FTP_TLS) -> list[str]: - return [x[0] for x in ftp.mlsd()] - - -def create_release_folder(ftp: FTP_TLS, tag: str) -> None: - ftp.cwd(RELEASE_FOLDER_BASE) - folder = get_release_folder_from_tag(tag) - try: - ftp.mkd(folder) - except error_perm: - pass - ftp.cwd(folder) - - -def upload_all_from_dir(ftp: FTP_TLS, filedir: Path) -> None: - for filepath in filedir.iterdir(): - upload_file(ftp, filepath) - - -def upload_file(ftp: FTP_TLS, - filepath: Path, - name: Optional[str] = None) -> None: - - if name is None: - name = filepath.name - - log.info('Upload file %s as %s', filepath.name, name) - with open(filepath, 'rb') as f: - ftp.storbinary('STOR ' + name, f) - - -def download_artifacts(path: Path) -> None: - build_results = list(path.glob('*.json')) - if not build_results: - sys.exit('No build build_results found') - - responses = [json.loads(response.read_text()) for response in build_results] - - for response in responses: - for artifact in response: - filename = cast(str, artifact['fileName']) - file_url = artifact['fileUrl'] - - log.info('Download %s', filename) - - req = requests.get(file_url, headers=HEADERS) - req.raise_for_status() - with open(path / filename, 'wb') as file: - file.write(req.content) - - for result in build_results: - result.unlink() - - -def get_deploy_method() -> str: - deploy_type = os.environ['DEPLOY_TYPE'] - is_nightly = bool(os.environ.get('GAJIM_NIGHTLY_BUILD')) - if is_nightly: - return f'deploy_{deploy_type}_nightly' - return f'deploy_{deploy_type}_release' - - -@ftp_connection -def deploy_windows_nightly(ftp: FTP_TLS, filedir: Path) -> None: - ftp.cwd(WINDOWS_NIGHTLY_FOLDER) - download_artifacts(filedir) - upload_all_from_dir(ftp, filedir) - - -@ftp_connection -def deploy_windows_release(ftp: FTP_TLS, filedir: Path) -> None: - tag = get_gajim_tag() - create_release_folder(ftp, tag) - download_artifacts(filedir) - upload_all_from_dir(ftp, filedir) - - -@ftp_connection -def deploy_linux_nightly(ftp: FTP_TLS, filedir: Path) -> None: - ftp.cwd(LINUX_NIGHTLY_FOLDER) - filepath = find_linux_tarball(filedir) - filename = f'gajim-{date.today().isoformat()}.tar.gz' - upload_file(ftp, filepath, name=filename) - - -@ftp_connection -def deploy_linux_release(ftp: FTP_TLS, file: Path) -> None: - tag = get_gajim_tag() - create_release_folder(ftp, tag) - filepath = find_linux_tarball(filedir) - filename = f'gajim-{tag}.tar.gz' - upload_file(ftp, filepath, name=filename) - - -if __name__ == '__main__': - filedir = Path(sys.argv[1]) - current_module = sys.modules[__name__] - method = getattr(current_module, get_deploy_method()) - method(filedir)

===================================== .gitlab-ci.yml ===================================== @@ -2,7 +2,7 @@ image: gajim-master:latest

workflow: rules: - - if: $GAJIM_NIGHTLY_BUILD + - if: $NIGHTLY_BUILD - if: $CI_PIPELINE_SOURCE == "push"

stages: @@ -16,8 +16,8 @@ test-other: script: - codespell gajim - pip3 install build - - python3 -m build -w - - appstream-util validate dist/share/metainfo/org.gajim.Gajim.appdata.xml + - python3 -m build + - appstream-util validate dist/metadata/org.gajim.Gajim.appdata.xml interruptible: true

test-pyright: @@ -68,62 +68,162 @@ test-unittest: path: coverage.xml interruptible: true

-build-linux: +build-unix: image: gajim-deb-build:latest stage: build dependencies: [] rules: - - if: '$GAJIM_NIGHTLY_BUILD' + - if: '$NIGHTLY_BUILD' - if: '$CI_COMMIT_TAG' script: - - apt-get update - - apt-get install -y python3-nbxmpp-nightly - pip3 install build - - python3 -m build -s - - python3 .ci/debian_build.py "$(find dist/gajim-*)" 1 + - python3 -m build

artifacts: name: "gajim-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA" expire_in: 1 week paths: - dist/gajim-*.tar.gz + - dist/gajim-*.whl + +build-debian-nightly: + image: gajim-deb-build:latest + stage: build + dependencies: + - build-unix + rules: + - if: '$NIGHTLY_BUILD' + script: + - apt-get update + - apt-get install -y python3-nbxmpp-nightly + - release-helper build-debian-pkg "$(find dist/gajim-*.tar.gz)" 1 --pkgsuffix=nightly + + artifacts: + name: "gajim-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA" + expire_in: 1 week + paths: + - debian_build/*

build-windows: stage: build dependencies: [] rules: - - if: '$GAJIM_NIGHTLY_BUILD' - if: '$CI_COMMIT_TAG' script: - - python3 .ci/appveyor_build.py + - > + release-helper build-windows-pkg \ + --account=lovetox \ + --slug=gajim \ + --branch=master \ + --tag=$CI_COMMIT_TAG \ + --commit-sha=$CI_COMMIT_SHA \ + --api-key=$APPVEYOR_API_KEY \ + .ci/appveyor.yml + artifacts: + expire_in: 1 day + paths: + - windows_build/*.json + +build-windows-nightly: + stage: build + dependencies: [] + rules: + - if: '$NIGHTLY_BUILD' + script: + - > + release-helper build-windows-pkg \ + --account=lovetox \ + --slug=gajim \ + --branch=master \ + --tag=Nightly \ + --commit-sha=$CI_COMMIT_SHA \ + --api-key=$APPVEYOR_API_KEY \ + .ci/appveyor.yml artifacts: expire_in: 1 day paths: - - build/*.json + - windows_build/*.json

-deploy-linux: +deploy-unix: stage: deploy dependencies: - - build-linux + - build-unix rules: - - if: '$GAJIM_NIGHTLY_BUILD' - if: '$CI_COMMIT_TAG' - variables: - DEPLOY_TYPE: "linux" script: - - python3 .ci/deploy.py dist + - > + release-helper deploy-to-ftp \ + --host=$FTP_HOST \ + --user=$FTP_USER \ + --password=$FTP_PASS \ + --directory=downloads/"$(echo $CI_COMMIT_TAG | cut -c 1-3)" \ + --rename=gajim-$CI_COMMIT_TAG.tar.gz \ + "$(find dist/gajim-*.tar.gz)" + +deploy-unix-nightly: + stage: deploy + dependencies: + - build-unix + rules: + - if: '$NIGHTLY_BUILD' + script: + - > + release-helper deploy-to-ftp \ + --host=$FTP_HOST \ + --user=$FTP_USER \ + --password=$FTP_PASS \ + --directory=downloads/snap" \ + --rename=gajim-"$(date +'%Y-%m-%d')".tar.gz \ + "$(find dist/gajim-*.tar.gz)" + +deploy-debian-nightly: + stage: deploy + dependencies: + - build-debian-nightly + rules: + - if: '$NIGHTLY_BUILD' + script: + - > + release-helper deploy-to-ftp \ + --host=$FTP_HOST \ + --user=$FTP_USER \ + --password=$FTP_PASS \ + --directory=debian/gajim/"$(date +'%Y%m%d')" \ + debian_build

deploy-windows: stage: deploy dependencies: - build-windows rules: - - if: '$GAJIM_NIGHTLY_BUILD' - if: '$CI_COMMIT_TAG' - variables: - DEPLOY_TYPE: "windows" script: - - python3 .ci/deploy.py build + - release-helper download-appveyor-artifacts "$(find windows_build/*.json)" windows_build + - rm windows_build/*.json + - > + release-helper deploy-to-ftp \ + --host=$FTP_HOST \ + --user=$FTP_USER \ + --password=$FTP_PASS \ + --directory=downloads/"$(echo $CI_COMMIT_TAG | cut -c 1-3)" \ + windows_build + +deploy-windows-nightly: + stage: deploy + dependencies: + - build-windows-nightly + rules: + - if: '$NIGHTLY_BUILD' + script: + - release-helper download-appveyor-artifacts "$(find windows_build/*.json)" windows_build + - rm windows_build/*.json + - > + release-helper deploy-to-ftp \ + --host=$FTP_HOST \ + --user=$FTP_USER \ + --password=$FTP_PASS \ + --directory=downloads/snap" \ + windows_build

deploy-flatpak: image: git-deploy:latest

===================================== MANIFEST.in ===================================== @@ -3,4 +3,5 @@ include ChangeLog recursive-include pep517build * recursive-include po * recursive-include data *.1 *.in -recursive-include test * \ No newline at end of file +recursive-include test * +prune */__pycache__

===================================== debian/control ===================================== @@ -65,6 +65,7 @@ Recommends: python3-sentry-sdk, Suggests: libxss1, +Breaks: gajim Description: Fully-featured XMPP chat client Gajim aims to be an easy to use and fully-featured XMPP client. Just chat with your friends or family, easily share pictures and

View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/29e8379aeb0ed1122738782c078c3d7a...