[Git][gajim/gajim][master] 2 commits: ci: Split windows job into build and deploy stages - Commits

29 Jul 2022


      Philipp Hörist pushed to branch master at gajim / gajim
Commits:
54d4c9f8 by lovetox at 2022-07-29T23:04:43+02:00
ci: Split windows job into build and deploy stages
- - - - -
f5d7ee23 by lovetox at 2022-07-29T23:04:43+02:00
ci: Move git setup into script
- - - - -
4 changed files:
- .ci/appveyor_build.py
- .ci/deploy.py
- + .ci/prepare_git.sh
- .gitlab-ci.yml
Changes:
=====================================
.ci/appveyor_build.py
=====================================
@@ -1,5 +1,8 @@
 #!/usr/bin/env python3
+from typing import Any
+
+import json
 import os
 import requests
 import time
@@ -7,6 +10,7 @@
from rich.console import Console
+
 ACCOUNT = 'lovetox'
 PROJECT_SLUG = 'gajim'
 BRANCH = 'master'
@@ -67,7 +71,7 @@ def is_build_finished(build: dict[str, str]) -> bool:
     return build['status'] == 'success'
-def get_artifacts(build_id: str) -> None:
+def check_for_response(build_id: str) -> None:
     time.sleep(INITIAL_START_DELAY)
     while True:
         time.sleep(RETRY_TIMEOUT)
@@ -89,28 +93,26 @@ def get_artifacts(build_id: str) -> None:
     build_folder.mkdir()
for job in build['jobs']:
-        download_job_artifacts(job['jobId'], build_folder)
-
-    console.print('All artifacts downloaded!')
+        response = get_job_response(job['jobId'])
+        result = build_folder / f'{job["jobId"]}.json'
+        result.write_text(json.dumps(response))
+        console.print('Write job response:', result)
-def download_job_artifacts(job_id: str, target_folder: Path) -> None:
+def get_job_response(job_id: str) -> list[dict[str, Any]]:
     artifacts_api_url = f'{BASE_URL}/buildjobs/{job_id}/artifacts'
     req = requests.get(artifacts_api_url, headers=HEADERS)
     req.raise_for_status()
     response = req.json()
for artifact in response:
-        filename = artifact['fileName']
-        console.print('Download', filename, '...')
-        file_url = f'{artifacts_api_url}/{filename}'
-        req = requests.get(file_url, headers=HEADERS)
-        req.raise_for_status()
-        with open(target_folder / filename, 'wb') as file:
-            file.write(req.content)
+        file_url = f'{artifacts_api_url}/{artifact["fileName"]}'
+        artifact['fileUrl'] = file_url
+
+    return response
if __name__ == '__main__':
     push_yaml_to_project()
     build_id = start_build()
-    get_artifacts(build_id)
+    check_for_response(build_id)
=====================================
.ci/deploy.py
=====================================
@@ -1,13 +1,17 @@
+#!/usr/bin/env python3
+
 from typing import Any
 from typing import Optional
import functools
+import json
 import os
 import sys
 from datetime import date
 from ftplib import FTP_TLS
 from pathlib import Path
+import requests
 from rich.console import Console
@@ -15,6 +19,9 @@
 FTP_USER = os.environ['FTP_USER']
 FTP_PASS = os.environ['FTP_PASS']
+API_KEY = os.environ['APPVEYOR_API_KEY']
+HEADERS = {'Authorization': f'Bearer {API_KEY}'}
+
 WINDOWS_NIGHTLY_FOLDER = 'downloads/snap/win'
 LINUX_NIGHTLY_FOLDER = 'downloads/snap'
@@ -84,6 +91,29 @@ def upload_file(ftp: FTP_TLS,
         ftp.storbinary('STOR ' + name, f)
+def download_artifacts(path: Path) -> None:
+    build_results = list(path.glob('*.json'))
+    if not build_results:
+        sys.exit('No build build_results found')
+
+    responses = [json.loads(response.read_text()) for response in build_results]
+
+    for response in responses:
+        for artifact in response:
+            filename = artifact['fileName']
+            file_url = artifact['fileUrl']
+
+            console.print('Download', filename, '...')
+
+            req = requests.get(file_url, headers=HEADERS)
+            req.raise_for_status()
+            with open(path / filename, 'wb') as file:
+                file.write(req.content)
+
+    for result in build_results:
+        result.unlink()
+
+
 def get_deploy_method() -> str:
     deploy_type = os.environ['DEPLOY_TYPE']
     is_nightly = bool(os.environ.get('GAJIM_NIGHTLY_BUILD'))
@@ -95,6 +125,7 @@ def get_deploy_method() -> str:
 @ftp_connection
 def deploy_windows_nightly(ftp: FTP_TLS, filedir: Path) -> None:
     ftp.cwd(WINDOWS_NIGHTLY_FOLDER)
+    download_artifacts(filedir)
     upload_all_from_dir(ftp, filedir)
@@ -102,6 +133,7 @@ def deploy_windows_nightly(ftp: FTP_TLS, filedir: Path) -> None:
 def deploy_windows_release(ftp: FTP_TLS, filedir: Path) -> None:
     tag = get_gajim_tag()
     create_release_folder(ftp, tag)
+    download_artifacts(filedir)
     upload_all_from_dir(ftp, filedir)
=====================================
.ci/prepare_git.sh
=====================================
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+eval $(ssh-agent -s)
+echo "$2" | tr -d '\r' | ssh-add - > /dev/null
+mkdir -p ~/.ssh
+chmod 700 ~/.ssh
+ssh-keyscan "$1" >> ~/.ssh/known_hosts
+git config --global user.email "$CI_GIT_USER_EMAIL"
+git config --global user.name "$CI_GIT_USER_USERNAME"
=====================================
.gitlab-ci.yml
=====================================
@@ -84,22 +84,18 @@ build-linux:
     paths:
       - dist/gajim-*.tar.gz
-build-and-deploy-windows:
+build-windows:
   stage: build
   dependencies: []
   rules:
     - if: '$GAJIM_NIGHTLY_BUILD'
     - if: '$CI_COMMIT_TAG'
-  variables:
-    DEPLOY_TYPE: "windows"
   script:
     - python3 .ci/appveyor_build.py
-    - python3 .ci/deploy.py build
-
-  # artifacts:
-  #   expire_in: 1 day
-  #   paths:
-  #     - build/Gajim-*.exe
+  artifacts:
+    expire_in: 1 day
+    paths:
+      - build/*.json
deploy-linux:
   stage: deploy
@@ -113,16 +109,17 @@ deploy-linux:
   script:
     - python3 .ci/deploy.py dist
-# deploy-windows:
-#   stage: deploy
-#   rules:
-#     - if: '$GAJIM_NIGHTLY_BUILD'
-#     - if: '$CI_COMMIT_TAG'
-#       when: manual
-#   variables:
-#     DEPLOY_TYPE: "windows"
-#   script:
-#     - python3 .ci/deploy.py build
+deploy-windows:
+  stage: deploy
+  dependencies:
+    - build-windows
+  rules:
+    - if: '$GAJIM_NIGHTLY_BUILD'
+    - if: '$CI_COMMIT_TAG'
+  variables:
+    DEPLOY_TYPE: "windows"
+  script:
+    - python3 .ci/deploy.py build
deploy-flatpak:
   image: git-deploy:latest
@@ -131,14 +128,8 @@ deploy-flatpak:
   rules:
     - if: '$CI_COMMIT_TAG'
   before_script:
-    - eval $(ssh-agent -s)
-    - echo "${FLATHUB_DEPLOY_KEY}" | tr -d '\r' | ssh-add - > /dev/null
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - ssh-keyscan github.com >> ~/.ssh/known_hosts
+    - .ci/prepare_git.sh github.com ${FLATHUB_DEPLOY_KEY}
   script:
-    - git config --global user.email "$CI_GIT_USER_EMAIL"
-    - git config --global user.name "$CI_GIT_USER_USERNAME"
     - git clone git@github.com:flathub/org.gajim.Gajim.git
     - cd org.gajim.Gajim
     - mv ../flatpak/org.gajim.Gajim.yaml org.gajim.Gajim.yaml
@@ -155,14 +146,8 @@ publish-release:
   rules:
     - if: '$CI_COMMIT_TAG'
   before_script:
-    - eval $(ssh-agent -s)
-    - echo "${WEBSITE_DEPLOY_KEY}" | tr -d '\r' | ssh-add - > /dev/null
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - ssh-keyscan dev.gajim.org >> ~/.ssh/known_hosts
+    - .ci/prepare_git.sh dev.gajim.org ${WEBSITE_DEPLOY_KEY}
   script:
-    - git config --global user.email "$CI_GIT_USER_EMAIL"
-    - git config --global user.name "$CI_GIT_USER_USERNAME"
     - git clone git@dev.gajim.org:gajim/website.git
     - cd website
     - git checkout master
View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/b7cd7dcc3140044dddd1d67a7243a829...
-- 
View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/b7cd7dcc3140044dddd1d67a7243a829...
You're receiving this email because of your account on dev.gajim.org.





    