Commits

commits@gajim.org

17632 discussions

[Git][gajim/gajim][soup3] change: Port to libsoup3
by Philipp Hörist (＠lovetox) 28 Dec '22

28 Dec '22

Philipp Hörist pushed to branch soup3 at gajim / gajim Commits: 747c5696 by Philipp Hörist at 2022-12-28T16:23:51+01:00 change: Port to libsoup3 - Remove all direct usage of libsoup and use the nbxmpp.http module instead - - - - - 15 changed files: - .gitlab-ci.yml - README.md - gajim/common/application.py - gajim/common/client.py - gajim/common/filetransfer.py - gajim/common/helpers.py - gajim/common/modules/httpupload.py - gajim/common/preview.py - + gajim/common/util/http.py - gajim/gajim.py - gajim/gtk/conversation/rows/file_transfer.py - gajim/gtk/exception.py - gajim/plugins/repository.py - win/_base.sh - win/dev_env.sh Changes: ===================================== .gitlab-ci.yml ===================================== @@ -26,7 +26,7 @@ test-pyright: - "**/*.py" script: - pip3 install git+https://dev.gajim.org/gajim/python-nbxmpp.git - - PYGOBJECT_STUB_CONFIG=Gtk3,Gdk3,Soup2 pip3 install git+https://github.com/pygobject/pygobject-stubs.git + - PYGOBJECT_STUB_CONFIG=Gtk3,Gdk3 pip3 install git+https://github.com/pygobject/pygobject-stubs.git - npm install pyright - python3 .ci/link-gtk.py - $(npm bin)/pyright --version ===================================== README.md ===================================== @@ -18,7 +18,6 @@ ### Runtime Requirements - [GLib](https://gitlab.com/gnome/glib) (>=2.60.0) - [GtkSourceView](https://gitlab.gnome.org/GNOME/gtksourceview) - [Pango](https://gitlab.gnome.org/GNOME/pango) (>=1.50.0) -- [libsoup](https://gitlab.gnome.org/GNOME/libsoup/) - [sqlite](https://www.sqlite.org/) (>=3.33.0) ### Optional Runtime Requirements ===================================== gajim/common/application.py ===================================== @@ -29,9 +29,9 @@ from gi.repository import Gio from gi.repository import GLib -from gi.repository import Soup from nbxmpp.const import ConnectionType from nbxmpp.const import ConnectionProtocol +from nbxmpp.http import HTTPRequest from gajim.common import app from gajim.common import ged @@ -48,7 +48,6 @@ from gajim.common.events import GajimUpdateAvailable from gajim.common.events import SignedIn from gajim.common.client import Client -from gajim.common.helpers import make_http_request from gajim.common.helpers import get_random_string from gajim.common.helpers import get_global_show from gajim.common.helpers import from_one_line @@ -60,6 +59,7 @@ from gajim.common.storage.cache import CacheStorage from gajim.common.storage.draft import DraftStorage from gajim.common.storage.archive import MessageArchiveStorage +from gajim.common.util.http import create_http_request from gajim.plugins.pluginmanager import PluginManager from gajim.plugins.repository import PluginRepository @@ -310,22 +310,21 @@ def _check_for_updates(self) -> None: def check_for_gajim_updates(self) -> None: self._log.info('Checking for Gajim updates') - make_http_request('https://gajim.org/current-version.json', - self._on_update_response) + request = create_http_request() + request.send('GET', 'https://gajim.org/current-version.json', + callback=self._on_update_response) - def _on_update_response(self, - _session: Soup.Session, - message: Soup.Message) -> None: + def _on_update_response(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + self._log.warning('Error while performing gajim update check: %s', + request.get_error(), request.get_status()) + return now = datetime.now() app.settings.set('last_update_check', now.strftime('%Y-%m-%d %H:%M')) - response_body = message.props.response_body - if response_body is None or not response_body.data: - self._log.warning('Could not reach gajim.org for update check') - return - - data = json.loads(response_body.data) + data = json.loads(request.get_data()) latest_version = data['current_version'] if V(latest_version) > V(app.version): ===================================== gajim/common/client.py ===================================== @@ -55,6 +55,7 @@ from gajim.common.idle import IdleMonitorManager from gajim.common.i18n import _ from gajim.common.structs import OutgoingMessage +from gajim.common.util.http import create_http_session from gajim.gui.util import open_window @@ -175,6 +176,8 @@ def _create_client(self) -> None: self._client.set_password(self.password) + self._client.set_http_session(create_http_session()) + self._client.subscribe('resume-failed', self._on_resume_failed) self._client.subscribe('resume-successful', self._on_resume_successful) self._client.subscribe('disconnected', self._on_disconnected) ===================================== gajim/common/filetransfer.py ===================================== @@ -27,8 +27,7 @@ def __init__(self, account: str) -> None: self._account = account - self._seen: int = 0 - self.size: int = 0 + self._progress = 0 self._state = FTState.INIT self._error_text: str = '' @@ -42,16 +41,6 @@ def account(self) -> str: def state(self) -> FTState: return self._state - @property - def seen(self) -> int: - return self._seen - - @property - def is_complete(self) -> bool: - if self.size == 0: - return False - return self._seen >= self.size - @property def filename(self) -> str: raise NotImplementedError @@ -64,6 +53,13 @@ def error_text(self) -> str: def error_domain(self) -> Optional[str]: return self._error_domain + def get_progress(self) -> float: + return self._progress + + def set_progress(self, progress: float) -> None: + self._progress = progress + self.update_progress() + def get_state_description(self) -> str: if self._state is None: return '' ===================================== gajim/common/helpers.py ===================================== @@ -81,7 +81,6 @@ from gi.repository import Gio from gi.repository import GLib from gi.repository import GObject -from gi.repository import Soup import precis_i18n.codec # noqa: F401 @@ -1546,24 +1545,3 @@ def make_path_from_jid(base_path: Path, jid: JID) -> Path: if jid.resource is not None: return path / jid.resource[:30] return path - - -def make_http_request(uri: str, - callback: Any, - user_data: Optional[Any] = None) -> None: - - proxy = determine_proxy() - if proxy is None: - resolver = None - - else: - resolver = proxy.get_resolver() - - session = Soup.Session() - session.props.proxy_resolver = resolver - session.props.user_agent = f'Gajim {app.version}' - message = Soup.Message.new('GET', uri) - if user_data is None: - session.queue_message(message, callback) - else: - session.queue_message(message, callback, user_data) ===================================== gajim/common/modules/httpupload.py ===================================== @@ -20,12 +20,13 @@ from typing import Callable from typing import Optional -import os -import io -from urllib.parse import urlparse -import mimetypes from collections import defaultdict +import functools from pathlib import Path +from urllib.parse import urlparse +import mimetypes +import os +import tempfile from nbxmpp.errors import StanzaError from nbxmpp.errors import MalformedStanzaError @@ -36,23 +37,25 @@ from nbxmpp.structs import HTTPUploadData from nbxmpp.task import Task from nbxmpp.util import convert_tls_error_flags +from nbxmpp.http import HTTPRequest +from nbxmpp.const import HTTPRequestError from gi.repository import Gio from gi.repository import GLib -from gi.repository import Soup from gajim.common import app from gajim.common import types from gajim.common.events import HTTPUploadError from gajim.common.events import HTTPUploadStarted from gajim.common.i18n import _ +from gajim.common.helpers import get_random_string from gajim.common.helpers import get_tls_error_phrases -from gajim.common.helpers import get_account_proxy from gajim.common.const import FTState from gajim.common.filetransfer import FileTransfer from gajim.common.modules.base import BaseModule from gajim.common.exceptions import FileError from gajim.common.structs import OutgoingMessage +from gajim.common.util.http import create_http_request class HTTPUpload(BaseModule): @@ -67,24 +70,11 @@ def __init__(self, con: types.Client) -> None: self.httpupload_namespace: Optional[str] = None self.max_file_size: Optional[float] = None # max file size in bytes - self._proxy_resolver: Optional[Gio.SimpleProxyResolver] = None - self._queued_messages: dict[int, Soup.Message] = {} - self._session = Soup.Session() - self._session.props.ssl_strict = False - self._session.props.user_agent = f'Gajim {app.version}' + self._requests_in_progress: dict[int, HTTPRequest] = {} self._running_transfers: dict[ tuple[str, JID], set[HTTPFileTransfer]] = defaultdict(set) - def _set_proxy_if_available(self) -> None: - proxy = get_account_proxy(self._account) - if proxy is None: - self._proxy_resolver = None - self._session.props.proxy_resolver = None - else: - self._proxy_resolver = proxy.get_resolver() - self._session.props.proxy_resolver = self._proxy_resolver - def pass_disco(self, info: DiscoInfo) -> None: if not info.has_httpupload: return @@ -215,11 +205,11 @@ def _on_cancel_upload(self, key = (transfer.account, transfer.contact.jid) self._running_transfers[key].discard(transfer) - message = self._queued_messages.get(id(transfer)) - if message is None: + request = self._requests_in_progress.get(id(transfer)) + if request is None: return - self._session.cancel_message(message, Soup.Status.CANCELLED) + request.cancel() def _start_transfer(self, transfer: HTTPFileTransfer) -> None: if transfer.encryption is not None and not transfer.is_encrypted: @@ -283,106 +273,61 @@ def _upload_file(self, transfer: HTTPFileTransfer) -> None: transfer.set_started() assert transfer.put_uri is not None - message = Soup.Message.new('PUT', transfer.put_uri) - message.connect('starting', self._check_certificate, transfer) - - # Set CAN_REBUILD so chunks get discarded after they have been - # written to the network - message.set_flags(Soup.MessageFlags.CAN_REBUILD | - Soup.MessageFlags.NO_REDIRECT) + request = create_http_request(self._account) + request.set_user_data(transfer) + request.connect('accept-certificate', self._accept_certificate) + request.connect('request-progress', self._on_request_progress) - assert message.props.request_body is not None - message.props.request_body.set_accumulate(False) + request.set_request_body_from_path(transfer.mime, transfer.payload_path) - assert message.props.request_headers is not None - message.props.request_headers.set_content_type(transfer.mime, None) - message.props.request_headers.set_content_length(transfer.size) for name, value in transfer.headers.items(): - message.props.request_headers.append(name, value) - - message.connect('wrote-headers', self._on_wrote_headers, transfer) - message.connect('wrote-chunk', self._on_wrote_chunk, transfer) - - self._queued_messages[id(transfer)] = message - self._set_proxy_if_available() - self._session.queue_message(message, self._on_finish, transfer) - - def _check_certificate(self, - message: Soup.Message, - transfer: HTTPFileTransfer - ) -> None: - https_used, tls_certificate, tls_errors = message.get_https_status() - if not https_used: - self._log.warning('HTTPS was not used for upload') - transfer.set_error('unsecure') - self._session.cancel_message(message, Soup.Status.CANCELLED) - return + request.get_request_headers().append(name, value) - tls_error_set = convert_tls_error_flags(tls_errors) - if app.cert_store.verify(tls_certificate, tls_error_set): - return + request.send('PUT', transfer.put_uri, callback=self._on_finish) + + self._requests_in_progress[id(transfer)] = request + + def _accept_certificate(self, + request: HTTPRequest, + _certificate: Gio.TlsCertificate, + certificate_errors: Gio.TlsCertificateFlags, + ) -> bool: - phrases = get_tls_error_phrases(tls_error_set) + transfer = request.get_user_data() + phrases = get_tls_error_phrases( + convert_tls_error_flags(certificate_errors)) self._log.warning( - 'TLS verification failed: %s (0x%02x)', phrases, tls_errors) + 'TLS verification failed: %s (0x%02x)', phrases, certificate_errors) + transfer.set_error('tls-verification-failed', phrases[0]) - self._session.cancel_message(message, Soup.Status.CANCELLED) + return False - def _on_finish(self, - _session: Soup.Session, - message: Soup.Message, - transfer: HTTPFileTransfer - ) -> None: + def _on_finish(self, request: HTTPRequest) -> None: + transfer = request.get_user_data() - self._queued_messages.pop(id(transfer), None) + self._requests_in_progress.pop(id(transfer), None) key = (transfer.account, transfer.contact.jid) self._running_transfers[key].discard(transfer) - if message.props.status_code == Soup.Status.CANCELLED: - self._log.info('Upload cancelled') - return - - if message.props.status_code in (Soup.Status.OK, Soup.Status.CREATED): - self._log.info('Upload completed successfully') - transfer.set_finished() - - else: - phrase = Soup.Status.get_phrase(message.props.status_code) - self._log.error('Got unexpected http upload response code: %s', - phrase) - transfer.set_error('http-response', phrase) - - def _on_wrote_chunk(self, - message: Soup.Message, - transfer: HTTPFileTransfer - ) -> None: - transfer.update_progress() - if transfer.is_complete: - assert message.props.request_body is not None - message.props.request_body.complete() + if not request.is_complete(): + error = request.get_error_string() + if request.get_error() == HTTPRequestError.CANCELLED: + self._log.info('Upload cancelled') + else: + transfer.set_error('http-response', error) return - bytes_ = transfer.get_chunk() - assert bytes_ is not None - self._session.pause_message(message) - GLib.idle_add(self._append, message, bytes_) + transfer.set_finished() + self._log.info('Upload completed successfully') - def _append(self, message: Soup.Message, bytes_: bytes) -> None: - if message.props.status_code == Soup.Status.CANCELLED: - return - self._session.unpause_message(message) - assert message.props.request_body is not None - message.props.request_body.append(bytes_) + def _on_request_progress(self, + request: HTTPRequest, + progress: float + ) -> None: - @staticmethod - def _on_wrote_headers(message: Soup.Message, - transfer: HTTPFileTransfer - ) -> None: - bytes_ = transfer.get_chunk() - assert bytes_ is not None - assert message.props.request_body is not None - message.props.request_body.append(bytes_) + transfer = request.get_user_data() + transfer.set_progress(progress) class HTTPFileTransfer(FileTransfer): @@ -410,23 +355,32 @@ def __init__(self, FileTransfer.__init__(self, account) - self._path = path + self._path = Path(path) self._encryption = encryption self._groupchat = groupchat self._contact = contact self._mime = mime - self.size = os.stat(path).st_size self.put_uri: Optional[str] = None self.get_uri: Optional[str] = None self._uri_transform_func: Optional[Callable[[str], str]] = None - self._stream = None self._data: Optional[bytes] = None self._headers: dict[str, str] = {} self._is_encrypted = False + @property + def size(self) -> int: + if self._encryption is not None and not self._is_encrypted: + raise ValueError('File size unknown at this point') + return self.payload_path.stat().st_size + + @size.setter + def size(self, size: int) -> None: + # Backwards compatibility with plugins + pass + @property def mime(self) -> str: return self._mime @@ -448,7 +402,13 @@ def headers(self) -> dict[str, str]: return self._headers @property - def path(self) -> str: + def path(self) -> Path: + return self._path + + @property + def payload_path(self) -> Path: + if self._encryption is not None: + return self.get_temp_path() return self._path @property @@ -465,47 +425,30 @@ def set_uri_transform_func(self, func: Callable[[str], str]) -> None: @property def filename(self) -> str: - return os.path.basename(self._path) + return self._path.name + + @staticmethod + @functools.cache + def get_temp_path() -> Path: + tempdir = tempfile.gettempdir() + return Path(tempdir) / get_random_string(16) def set_error(self, domain: str, text: str = '') -> None: if not text: text = self._errors[domain] - self._close() super().set_error(domain, text) def set_finished(self) -> None: - self._close() super().set_finished() def set_encrypted_data(self, data: bytes) -> None: - self._data = data + temp_path = self.get_temp_path() + temp_path.write_bytes(data) self._is_encrypted = True - def _close(self) -> None: - if self._stream is not None: - self._stream.close() - - def get_chunk(self) -> Optional[bytes]: - if self._stream is None: - if self._encryption is None: - self._stream = open(self._path, 'rb') # pylint: disable=consider-using-with # noqa: E501 - else: - self._stream = io.BytesIO(self._data) - - data = self._stream.read(16384) - if not data: - self._close() - return None - self._seen += len(data) - if self.is_complete: - self._close() - return data - def get_data(self) -> bytes: - with open(self._path, 'rb') as file: - data = file.read() - return data + return self._path.read_bytes() def process_result(self, result: HTTPUploadData) -> None: self.put_uri = result.put_uri ===================================== gajim/common/preview.py ===================================== @@ -14,12 +14,14 @@ from __future__ import annotations -from dataclasses import dataclass, field from typing import Any +from typing import cast from typing import Callable from typing import Optional import logging +from dataclasses import dataclass +from dataclasses import field import os import re from pathlib import Path @@ -29,7 +31,8 @@ from gi.repository import Gio from gi.repository import GLib -from gi.repository import Soup +from nbxmpp.const import HTTPRequestError +from nbxmpp.http import HTTPRequest from nbxmpp.util import convert_tls_error_flags from gajim.common import app @@ -40,7 +43,6 @@ from gajim.common.helpers import load_file_async from gajim.common.helpers import write_file_async from gajim.common.helpers import get_tls_error_phrases -from gajim.common.helpers import get_account_proxy from gajim.common.i18n import _ from gajim.common.preview_helpers import aes_decrypt from gajim.common.preview_helpers import filename_from_uri @@ -52,6 +54,7 @@ from gajim.common.preview_helpers import guess_mime_type from gajim.common.preview_helpers import pixbuf_from_data from gajim.common.types import GdkPixbufType +from gajim.common.util.http import create_http_request log = logging.getLogger('gajim.c.preview') @@ -104,12 +107,11 @@ def __init__(self, self.thumbnail: Optional[bytes] = None self.mime_type: str = '' self.file_size: int = 0 - self._received_size: int = 0 self.download_in_progress = False self.info_message: Optional[str] = None - self._soup_message: Optional[Soup.Message] = None + self._request = None self.key: Optional[bytes] = None self.iv: Optional[bytes] = None @@ -154,8 +156,8 @@ def filename(self) -> str: return self._filename @property - def soup_message(self) -> Soup.Message: - return self._soup_message + def request(self) -> HTTPRequest: + return self._request @property def request_uri(self) -> Optional[str]: @@ -193,29 +195,17 @@ def create_thumbnail(self, data: bytes) -> bool: return False return True - def reset_received_size(self) -> None: - self._received_size = 0 - def update_widget(self, data: Optional[GdkPixbufType] = None) -> None: self._widget.update(self, data) - def update_progress(self, size: int, message: Soup.Message) -> None: + def update_progress(self, progress: float, request: HTTPRequest) -> None: self.download_in_progress = True - self._soup_message = message - self._received_size += size - if self.file_size == 0 or self._received_size == 0: - return - - progress = self._received_size / self.file_size + self._request = request self._widget.update_progress(self, progress) class PreviewManager: def __init__(self) -> None: - self._sessions: dict[ - str, - tuple[Soup.Session, Optional[Gio.SimpleProxyResolver]]] = {} - self._orig_dir = Path(configpaths.get('MY_DATA')) / 'downloads' self._thumb_dir = Path(configpaths.get('MY_CACHE')) / 'downloads.thumb' @@ -268,29 +258,6 @@ def stop_audio_except(self, preview_id: int) -> None: if id_ != preview_id: stop_func() - def _get_session(self, account: str) -> Soup.Session: - if account not in self._sessions: - self._sessions[account] = self._create_session(account) - return self._sessions[account][0] - - @staticmethod - def _create_session(account: str) -> tuple[ - Soup.Session, Optional[Gio.SimpleProxyResolver]]: - session = Soup.Session() - session.add_feature_by_type(Soup.ContentSniffer) - session.props.https_aliases = ['aesgcm'] - session.props.ssl_strict = False - session.props.user_agent = f'Gajim {app.version}' - - proxy = get_account_proxy(account) - if proxy is None: - resolver = None - else: - resolver = proxy.get_resolver() - - session.props.proxy_resolver = resolver - return session, resolver - @staticmethod def _accept_uri(urlparts: ParseResult, uri: str, @@ -476,58 +443,62 @@ def download_content(self, log.info('Download already in progress') return - preview.reset_received_size() if preview.account is None: # History Window can be opened without account context # This means we can not apply proxy settings return log.info('Start downloading: %s', preview.request_uri) - message = Soup.Message.new('GET', preview.request_uri) - message.connect('starting', self._check_certificate, preview) - message.connect( - 'content-sniffed', self._on_content_sniffed, preview, force) - message.connect('got-chunk', self._on_got_chunk, preview) - session = self._get_session(preview.account) - session.queue_message(message, self._on_finished, preview) + request = create_http_request(preview.account, sniffer=True) + request.set_user_data(preview) + request.connect('accept-certificate', self._accept_certificate) + request.connect('content-sniffed', self._on_content_sniffed, force) + request.connect('response-progress', self._on_response_progress) - def _check_certificate(self, - message: Soup.Message, - preview: Preview) -> None: - _https_used, _tls_certificate, tls_errors = message.get_https_status() + request.send('GET', preview.request_uri, callback=self._on_finished) + + def _accept_certificate(self, + request: HTTPRequest, + certificate: Gio.TlsCertificate, + certificate_errors: Gio.TlsCertificateFlags, + ) -> bool: if not app.settings.get('preview_verify_https'): - return + return True - if tls_errors: - phrases = get_tls_error_phrases(convert_tls_error_flags(tls_errors)) - log.warning( - 'TLS verification failed: %s (0x%02x)', phrases, tls_errors) - session = self._get_session(preview.account) - session.cancel_message(message, Soup.Status.CANCELLED) - preview.info_message = _('TLS verification failed: %s') % phrases[0] - preview.update_widget() + phrases = get_tls_error_phrases( + convert_tls_error_flags(certificate_errors)) + log.warning( + 'TLS verification failed: %s (0x%02x)', phrases, certificate_errors) + + preview = cast(Preview, request.get_user_data()) + preview.info_message = _('TLS verification failed: %s') % phrases[0] + preview.update_widget() + return False def _on_content_sniffed(self, - message: Soup.Message, - type_: str, + request: HTTPRequest, + content_type: str, _params: GLib.HashTable, - preview: Preview, - force: bool) -> None: - file_size = message.props.response_headers.get_content_length() - uri = message.props.uri.to_string(False) - session = self._get_session(preview.account) - preview.mime_type = type_ + force: bool + ) -> None: + + file_size = request.get_response_headers().get_content_length() + uri = request.get_uri().to_string() + + + preview = cast(Preview, request.get_user_data()) + preview.mime_type = content_type preview.file_size = file_size - if type_ not in ALLOWED_MIME_TYPES and not force: - log.info('Not an allowed content type: %s, %s', type_, uri) - session.cancel_message(message, Soup.Status.CANCELLED) + if content_type not in ALLOWED_MIME_TYPES and not force: + log.info('Not an allowed content type: %s, %s', content_type, uri) + request.cancel() return if file_size == 0: log.info('File size is unknown (zero) for URL: "%s"', uri) - session.cancel_message(message, Soup.Status.CANCELLED) + request.cancel() return if file_size > int(app.settings.get('preview_max_file_size')): @@ -537,42 +508,37 @@ def _on_content_sniffed(self, if force: preview.info_message = None else: - session.cancel_message(message, Soup.Status.CANCELLED) + request.cancel() preview.info_message = _('Automatic preview disabled ' '(file too big)') preview.update_widget() - def _on_got_chunk(self, - message: Soup.Message, - chunk: Soup.Buffer, - preview: Preview - ) -> None: - preview.update_progress(len(chunk.get_data()), message) + def _on_response_progress(self, + request: HTTPRequest, + progress: float, + ) -> None: - def _on_finished(self, - _session: Soup.Session, - message: Soup.Message, - preview: Preview - ) -> None: + preview = cast(Preview, request.get_user_data()) + preview.update_progress(progress, request) + def _on_finished(self, request: HTTPRequest) -> None: + + preview = cast(Preview, request.get_user_data()) preview.download_in_progress = False - if message.status_code != Soup.Status.OK: - log.warning('Download failed: %s', preview.request_uri) - status_code = Soup.Status.get_phrase(message.status_code) - log.warning(status_code) - preview.reset_received_size() - if message.status_code != 1: - # status_code 1: 'Cancelled' - preview.info_message = _('Download failed (%s)') % status_code + if not request.is_complete(): + error = request.get_error_string() + log.warning('Download failed: %s - %s', preview.request_uri, error) + if request.get_error() != HTTPRequestError.CANCELLED: + preview.info_message = _('Download failed (%s)') % error preview.update_widget() return preview.info_message = None - data = message.props.response_body_data.get_data() - if data is None: + data = request.get_data() + if not data: return if preview.is_aes_encrypted: @@ -649,6 +615,4 @@ def _on_thumb_write_finished(_result: bool, preview.update_widget(data=pixbuf) def cancel_download(self, preview: Preview) -> None: - preview.reset_received_size() - session = self._get_session(preview.account) - session.cancel_message(preview.soup_message, Soup.Status.CANCELLED) + preview.request.cancel() ===================================== gajim/common/util/http.py ===================================== @@ -0,0 +1,48 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see <http://www.gnu.org/licenses/>. + +from typing import Optional + +from nbxmpp.http import HTTPRequest +from nbxmpp.http import HTTPSession + +from gajim.common import app +from gajim.common.helpers import determine_proxy +from gajim.common.helpers import get_account_proxy + + +def create_http_session(account: Optional[str] = None, + sniffer: bool = False + ) -> HTTPSession: + + session = HTTPSession(user_agent=f'Gajim {app.version}', + sniffer=sniffer) + + if account is None: + proxy = determine_proxy() + else: + proxy = get_account_proxy(account) + + if proxy is not None: + session.set_proxy_resolver(proxy.get_resolver()) + + return session + + +def create_http_request(account: Optional[str] = None, + sniffer: bool = False + ) -> HTTPRequest: + + session = create_http_session(account, sniffer) + return session.create_request() ===================================== gajim/gajim.py ===================================== @@ -65,8 +65,7 @@ def _check_required_deps() -> None: 'GtkSource': '4', 'GObject': '2.0', 'Pango': '1.0', - 'PangoCairo': '1.0', - 'Soup': '2.4'}) + 'PangoCairo': '1.0'}) except ValueError as error: sys.exit('Missing dependency: %s' % error) ===================================== gajim/gtk/conversation/rows/file_transfer.py ===================================== @@ -131,7 +131,10 @@ def _on_transfer_progress(self, size_total = GLib.format_size_full(transfer.size, self._units) self._ui.file_size.set_text(size_total) - bytes_sec = int(round(transfer.seen / (time_now - self._start_time), 1)) + progress = transfer.get_progress() + seen = transfer.size * progress + + bytes_sec = int(round(seen / (time_now - self._start_time), 1)) speed = f'{GLib.format_size_full(bytes_sec, self._units)}/s' self._ui.transfer_progress.set_tooltip_text(_('Speed: %s') % speed) @@ -139,9 +142,8 @@ def _on_transfer_progress(self, eta = '∞' else: eta = format_eta(round( - (transfer.size - transfer.seen) / bytes_sec)) + (transfer.size - seen) / bytes_sec)) - progress = float(transfer.seen) / transfer.size self._ui.transfer_progress.set_text( _('%(progress)s %% (%(time)s remaining)') % { 'progress': round(progress * 100), ===================================== gajim/gtk/exception.py ===================================== @@ -20,11 +20,11 @@ from __future__ import annotations from typing import Any -from types import TracebackType import sys import json import traceback +from types import TracebackType import threading import webbrowser import pprint @@ -33,9 +33,9 @@ from gi.repository import Gdk from gi.repository import Gtk -from gi.repository import Soup import nbxmpp +from nbxmpp.http import HTTPRequest import gajim from gajim.common import app @@ -44,8 +44,8 @@ from gajim.common.helpers import get_os_name from gajim.common.helpers import get_os_version from gajim.common.helpers import determine_proxy -from gajim.common.helpers import make_http_request from gajim.common.i18n import _ +from gajim.common.util.http import create_http_request from .builder import get_builder from .util import get_gtk_version @@ -182,24 +182,17 @@ def _request_sentry_endpoint(self) -> None: self._ui.report_spinner.show() self._ui.report_spinner.start() - make_http_request('https://gajim.org/updates.json', - self._on_endpoint_received) + request = create_http_request() + request.send('GET', 'https://gajim.org/updates.json', + callback=self._on_endpoint_received) - def _parse_endpoint(self, message: Soup.Message) -> str: - status = message.props.status_code - response_body = message.props.response_body - - if status != Soup.Status.OK: - error = message.props.reason_phrase + def _parse_endpoint(self, request: HTTPRequest) -> str: + if not request.is_complete(): raise ValueError('Failed to retrieve sentry endpoint: %s %s' % ( - status, error)) - - if response_body is None or not response_body.data: - raise ValueError('Failed to retrieve sentry endpoint, ' - 'no response body') + request.get_status(), request.get_error())) try: - data = json.loads(response_body.data) + data = json.loads(request.get_data()) except Exception as error: raise ValueError('Json parsing error: %s' % error) @@ -209,12 +202,9 @@ def _parse_endpoint(self, message: Soup.Message) -> str: return endpoint - def _on_endpoint_received(self, - _session: Soup.Session, - message: Soup.Message - ) -> None: + def _on_endpoint_received(self, request: HTTPRequest) -> None: try: - endpoint = self._parse_endpoint(message) + endpoint = self._parse_endpoint(request) except ValueError as error: print(error) self._report_with_browser() ===================================== gajim/plugins/repository.py ===================================== @@ -12,7 +12,7 @@ # You should have received a copy of the GNU General Public License # along with Gajim. If not, see <http://www.gnu.org/licenses/>. -from typing import Any +from typing import Any, cast from typing import Iterable from typing import Optional @@ -21,14 +21,14 @@ from io import BytesIO from zipfile import ZipFile -from gi.repository import Soup from gi.repository import GLib +from nbxmpp.http import HTTPRequest from gajim.common import app from gajim.common import configpaths from gajim.common.helpers import Observable -from gajim.common.helpers import make_http_request from gajim.plugins.manifest import PluginManifest +from gajim.common.util.http import create_http_request log = logging.getLogger('gajim.p.repository') @@ -60,28 +60,27 @@ def __init__(self) -> None: self._download_queue: set[PluginManifest] = set() if app.settings.get('plugins_repository_enabled'): - make_http_request('https://gajim.org/updates.json', - self._on_repository_received) + request = create_http_request() + request.send('GET', 'https://gajim.org/updates.json', + callback=self._on_repository_received) @property def available(self): return bool(self._repository_url) - def _on_repository_received(self, - _session: Soup.Session, - message: Soup.Message) -> None: + def _on_repository_received(self, request: HTTPRequest) -> None: - if message.status_code != Soup.Status.OK: - log.warning('Repository retrieval failed') - log.warning(Soup.Status.get_phrase(message.status_code)) + if not request.is_complete(): + log.warning('Repository retrieval failed: %s', + request.get_error_string()) return - assert message.props.response_body_data is not None - data = message.props.response_body_data.get_data() - if data is None: + try: + updates = json.loads(request.get_data()) + except Exception as error: + log.warning('Unable to parse repository information: %s', error) return - updates = json.loads(data.decode()) repository_url = updates.get('plugin_repository') if repository_url is None: log.warning('No plugin repository url found') @@ -148,56 +147,49 @@ def _parse_package_json(self, def _refresh_plugin_index(self, callback: Optional[Any] = None) -> None: log.info('Refresh index') - make_http_request(self._repository_index_url, - self._on_index_received, - callback) - - def _on_index_received(self, - _session: Soup.Session, - message: Soup.Message, - callback: Optional[Any] = None) -> None: - - if message.status_code != Soup.Status.OK: - log.warning('Refresh failed: %s', self._repository_index_url) - log.warning(Soup.Status.get_phrase(message.status_code)) - return + request = create_http_request() + request.set_user_data(callback) + request.send('GET', self._repository_index_url, + callback=self._on_index_received) - assert message.props.response_body_data is not None - data = message.props.response_body_data.get_data() - if data is None: + def _on_index_received(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + log.warning('Refresh failed: %s %s', + self._repository_index_url, + request.get_error_string()) return - package_index = json.loads(data.decode()) + try: + package_index = json.loads(request.get_data()) + except Exception as error: + log.warning('Unable to parse repository index: %s', error) + return self._manifests = self._parse_package_json(package_index) image_path = package_index['metadata']['image_path'] - make_http_request(f'{self._repository_url}/{image_path}', - self._on_images_received) + callback = request.get_user_data() + + request = create_http_request() + request.send('GET', f'{self._repository_url}/{image_path}', + callback=self._on_images_received) log.info('Refresh successful') if callback is not None: callback() - def _on_images_received(self, - _session: Soup.Session, - message: Soup.Message) -> None: + def _on_images_received(self, request: HTTPRequest) -> None: - if message.status_code != Soup.Status.OK: - log.warning('Image download failed') - log.warning(Soup.Status.get_phrase(message.status_code)) - return - - assert message.props.response_body_data is not None - data = message.props.response_body_data.get_data() - if data is None: + if not request.is_complete(): + log.warning('Image download failed: %s', request.get_error_string()) return path = configpaths.get('PLUGINS_IMAGES') try: - zipfile = ZipFile(BytesIO(data)) + zipfile = ZipFile(BytesIO(request.get_data())) zipfile.extractall(path) except Exception as error: log.warning('Failed to extract images: %s', error) @@ -245,27 +237,25 @@ def download_plugins(self, manifests: Iterable[PluginManifest]) -> None: def _download_plugin(self, manifest: PluginManifest) -> None: log.info('Download plugin %s', manifest.short_name) url = manifest.get_remote_url(self._repository_url) - make_http_request(url, self._on_download_plugin_finished, manifest) + request = create_http_request() + request.set_user_data(manifest) + request.send('GET', url, + callback=self._on_download_plugin_finished) - def _on_download_plugin_finished(self, - _session: Soup.Session, - message: Soup.Message, - manifest: PluginManifest) -> None: + def _on_download_plugin_finished(self, request: HTTPRequest) -> None: + manifest = cast(PluginManifest, request.get_user_data()) self._download_queue.remove(manifest) - if message.status_code != Soup.Status.OK: - error = Soup.Status.get_phrase(message.status_code) - log.error('Download failed: %s', - manifest.get_remote_url(self._repository_url)) - log.error(error) + if not request.is_complete(): + error = request.get_error_string() + log.error('Download failed: %s %s', + manifest.get_remote_url(self._repository_url), + error) self.notify('download-failed', manifest, error) return - assert message.props.response_body_data is not None - data = message.props.response_body_data.get_data() - - with ZipFile(BytesIO(data)) as zip_file: + with ZipFile(BytesIO(request.get_data())) as zip_file: zip_file.extractall(self._download_path / manifest.short_name) log.info('Finished downloading %s', manifest.short_name) ===================================== win/_base.sh ===================================== @@ -94,7 +94,7 @@ function install_deps { mingw-w64-"${ARCH}"-goocanvas \ mingw-w64-"${ARCH}"-gspell \ mingw-w64-"${ARCH}"-hunspell \ - mingw-w64-"${ARCH}"-libsoup \ + mingw-w64-"${ARCH}"-libsoup3 \ mingw-w64-"${ARCH}"-python-setuptools \ mingw-w64-"${ARCH}"-python-pillow \ mingw-w64-"${ARCH}"-python-setuptools-scm \ ===================================== win/dev_env.sh ===================================== @@ -22,7 +22,7 @@ function main { mingw-w64-x86_64-goocanvas \ mingw-w64-x86_64-gspell \ mingw-w64-x86_64-hunspell \ - mingw-w64-x86_64-libsoup \ + mingw-w64-x86_64-libsoup3 \ PIP_REQUIREMENTS="\ git+https://dev.gajim.org/gajim/python-nbxmpp.git View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/747c569638e0abb877517cee162854c7… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/747c569638e0abb877517cee162854c7… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/python-nbxmpp][master] change: Port to libsoup3
by Philipp Hörist (＠lovetox) 28 Dec '22

28 Dec '22

Philipp Hörist pushed to branch master at gajim / python-nbxmpp Commits: 0da8888d by Philipp Hörist at 2022-12-28T10:15:01+01:00 change: Port to libsoup3 - Port code to libsoup3 - Wrap Soup.Session and Soup.Message for easier usage - - - - - 15 changed files: - .gitlab-ci.yml - README.md - debian/control - nbxmpp/__init__.py - nbxmpp/addresses.py - nbxmpp/client.py - nbxmpp/const.py - nbxmpp/examples/client.py - + nbxmpp/http.py - nbxmpp/modules/muclumbus.py - nbxmpp/structs.py - nbxmpp/task.py - nbxmpp/websocket.py - + test/unit/test_http.py - + test/unit/test_muclumbus.py Changes: ===================================== .gitlab-ci.yml ===================================== @@ -9,7 +9,7 @@ test-pylint: stage: test script: - python3 .ci/pylint-test.py - - coverage run --source=nbxmpp -m unittest discover -v + - NBXMPP_EXTERNAL_UNIT_TESTS=1 coverage run --source=nbxmpp -m unittest discover -v - coverage report -mi - coverage xml -i - pip install . ===================================== README.md ===================================== @@ -7,7 +7,7 @@ - python >= 3.9.0 - PyGObject - GLib >= 2.60 -- libsoup +- libsoup3 - precis-i18n - packaging - idna ===================================== debian/control ===================================== @@ -6,7 +6,7 @@ Build-Depends: debhelper-compat (= 13), dh-python, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-all, python3-gi, python3-idna, @@ -24,7 +24,7 @@ Architecture: all Depends: ${misc:Depends}, ${python3:Depends}, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-gi, python3-idna, python3-packaging, ===================================== nbxmpp/__init__.py ===================================== @@ -2,6 +2,6 @@ import gi from .protocol import * -gi.require_version('Soup', '2.4') +gi.require_version('Soup', '3.0') __version__: str = '3.2.5' ===================================== nbxmpp/addresses.py ===================================== @@ -18,9 +18,8 @@ import logging from collections import namedtuple -from gi.repository import Soup - -import nbxmpp +from nbxmpp.http import HTTPRequest +from nbxmpp.http import HTTPSession from nbxmpp.util import Observable from nbxmpp.const import ConnectionType from nbxmpp.const import ConnectionProtocol @@ -63,6 +62,7 @@ class ServerAddresses(Observable): Observable.__init__(self, log) self._domain = domain + self._http_session = None self._custom_host = None self._proxy = None self._is_resolved = False @@ -136,32 +136,31 @@ class ServerAddresses(Observable): self._resolve_alternatives() def _resolve_alternatives(self) -> None: - session = Soup.Session() - session.props.timeout = 5 - session.props.user_agent = f'nbxmpp/{nbxmpp.__version__}' - message = Soup.Message.new( - 'GET', f'https://{self._domain}/.well-known/host-meta') - session.queue_message(message, self._on_alternatives_result) - - def _on_alternatives_result(self, - _session: Soup.Session, - message: Soup.Message) -> None: - - status = message.props.status_code - if status != Soup.Status.OK: - error = message.props.reason_phrase - log.info('Failed to retrieve host-meta file: %s %s', status, error) + if self._http_session is None: self._on_request_resolved() return - response_body = message.props.response_body - if response_body is None or not response_body.data: + request = self._http_session.create_request() + request.send('GET', f'https://{self._domain}/.well-known/host-meta', + timeout=5, + callback=self._on_alternatives_result) + + def _on_alternatives_result(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + log.info('Failed to retrieve host-meta file: %s', + request.get_error_string()) + self._on_request_resolved() + return + + response_body = request.get_data() + if not response_body: log.info('No response body data found') self._on_request_resolved() return try: - uri = parse_websocket_uri(response_body.data) + uri = parse_websocket_uri(response_body.decode()) except Exception as error: log.info('Error parsing websocket uri: %s', error) self._on_request_resolved() @@ -215,6 +214,9 @@ class ServerAddresses(Observable): def set_proxy(self, proxy): self._proxy = proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + def _on_request_resolved(self): self._is_resolved = True self.notify('resolved') ===================================== nbxmpp/client.py ===================================== @@ -22,6 +22,7 @@ from typing import Optional from gi.repository import GLib from nbxmpp.namespaces import Namespace +from nbxmpp.http import HTTPSession from nbxmpp.protocol import Features from nbxmpp.protocol import StanzaMalformed from nbxmpp.protocol import SessionRequest @@ -93,6 +94,7 @@ class Client(Observable): self._client_cert = None self._client_cert_pass = None self._proxy = None + self._http_session = HTTPSession() self._allowed_con_types = None self._allowed_protocols = None @@ -329,12 +331,18 @@ class Client(Observable): def set_proxy(self, proxy): self._proxy = proxy - self._dispatcher.get_module('Muclumbus').set_proxy(proxy) @property def proxy(self): return self._proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + + @property + def http_session(self) -> HTTPSession: + return self._http_session + def get_bound_jid(self): return self._jid @@ -402,6 +410,7 @@ class Client(Observable): self.state = StreamState.RESOLVE self._addresses = ServerAddresses(self._domain) + self._addresses.set_http_session(self._http_session) self._addresses.set_custom_host(self._custom_host) self._addresses.set_proxy(self._proxy) self._addresses.subscribe('resolved', self._on_addresses_resolved) ===================================== nbxmpp/const.py ===================================== @@ -453,6 +453,16 @@ class Mode(IntEnum): return self == Mode.ANONYMOUS_TEST +class HTTPRequestError(IntEnum): + UNKNOWN = 0 + MISSING_CONTENT_LENGTH = 1 + INCOMPLETE = 2 + STATUS_NOT_OK = 3 + CANCELLED = 4 + CONTENT_OVERFLOW = 5 + TIMEOUT = 6 + + MOODS = [ 'afraid', 'amazed', ===================================== nbxmpp/examples/client.py ===================================== @@ -6,6 +6,7 @@ import json from pathlib import Path import gi + gi.require_version('Gtk', '3.0') gi.require_version('GLib', '2.0') from gi.repository import Gtk @@ -14,6 +15,7 @@ from gi.repository import GLib import nbxmpp from nbxmpp.protocol import JID from nbxmpp.client import Client +from nbxmpp.http import HTTPSession from nbxmpp.structs import ProxyData from nbxmpp.structs import StanzaHandler from nbxmpp.const import ConnectionType @@ -83,6 +85,7 @@ class TestClient(Gtk.Window): self._client.set_domain(self.address.domain) self._client.set_username(self.address.localpart) self._client.set_resource('test') + self._client.set_http_session(HTTPSession()) proxy_ip = self._builder.proxy_ip.get_text() if proxy_ip: ===================================== nbxmpp/http.py ===================================== @@ -0,0 +1,474 @@ +# This file is part of nbxmpp. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see <http://www.gnu.org/licenses/>. + +from __future__ import annotations + +from pathlib import Path + +from typing import Any +from typing import cast +from typing import Literal +from typing import Callable +from typing import Optional + +import logging + +from gi.repository import Soup +from gi.repository import Gio +from gi.repository import GLib +from gi.repository import GObject + +import nbxmpp +from .const import HTTPRequestError + + +log = logging.getLogger('nbxmpp.http') + + +HTTP_METHODS_T = Literal[ + 'CONNECT', + 'DELETE', + 'GET', + 'HEAD', + 'OPTIONS', + 'PATCH', + 'POST', + 'PUT', + 'TRACE', +] +CHUNK_SIZE = 32768 +DEFAULT_USER_AGENT = f'nbxmpp/{nbxmpp.__version__}' +SIGNAL_ACTIONS = GObject.SignalFlags.RUN_LAST | GObject.SignalFlags.ACTION + + +class HTTPLogAdapter(logging.LoggerAdapter): + def process(self, msg: str, kwargs: Any) -> tuple[str, Any]: + return f'{self.extra["request"]}: {msg}', kwargs + + +class HTTPSession: + def __init__(self, + user_agent: str = DEFAULT_USER_AGENT, + sniffer: bool = False + ) -> None: + + self._session = Soup.Session() + self._session.set_user_agent(user_agent) + + if sniffer: + self._session.add_feature_by_type(Soup.ContentSniffer) + + def get_soup_session(self) -> Soup.Session: + return self._session + + def set_proxy_resolver(self, + resolver: Optional[Gio.SimpleProxyResolver] + ) -> None: + + self._session.set_proxy_resolver(resolver) + + def create_request(self) -> HTTPRequest: + return HTTPRequest(self) + + +class HTTPRequest(GObject.GObject): + + __gtype_name__ = "HTTPRequest" + + __gsignals__ = { + 'starting-response-body': (SIGNAL_ACTIONS, None, ()), + 'response-progress': (SIGNAL_ACTIONS, None, (float,)), + 'request-progress': (SIGNAL_ACTIONS, None, (float,)), + 'finished': (SIGNAL_ACTIONS, None, ()), + 'destroy': (SIGNAL_ACTIONS, None, ()), + } + + def __init__(self, session: HTTPSession) -> None: + GObject.GObject.__init__(self) + + self._log = HTTPLogAdapter(log, extra={'request': self}) + + self._session = session + + self._received_size = 0 + self._sent_size = 0 + + self._cancellable = Gio.Cancellable() + self._input_stream = cast(Gio.InputStream, None) + self._output_stream: Optional[Gio.OutputStream] = None + self._is_finished = False + self._error: Optional[HTTPRequestError] = None + self._is_complete = False + self._timeout_reached = False + self._timeout_id = None + + self._response_body_file: Optional[Gio.File] = None + self._response_body_data = b'' + + self._request_body_file: Optional[Gio.File] = None + self._request_body_data: Optional[bytes] = None + + self._request_content_type = '' + self._request_content_length = 0 + + self._response_content_type = '' + self._response_content_length = 0 + + self._emit_request_progress = False + self._emit_response_progress = False + + self._message = Soup.Message() + self._user_data = None + + self._log.info('Created') + + def is_finished(self) -> bool: + return self._is_finished + + def is_complete(self) -> bool: + return self._is_complete + + def get_error(self) -> Optional[HTTPRequestError]: + return self._error + + def get_error_string(self) -> str: + if self._error == HTTPRequestError.STATUS_NOT_OK: + return self._message.get_reason_phrase() + return f'{self._error}' + + def get_response_headers(self) -> Soup.MessageHeaders: + return self._message.get_response_headers() + + def get_request_headers(self) -> Soup.MessageHeaders: + return self._message.get_request_headers() + + def get_data(self) -> bytes: + if not self._is_finished: + raise ValueError('Process not finished, data not available') + return self._response_body_data + + def get_uri(self) -> Optional[GLib.Uri]: + return self._message.get_uri() + + def get_status(self) -> Soup.Status: + return self._message.get_status() + + def get_soup_message(self) -> Soup.Message: + return self._message + + def set_user_data(self, user_data: Any) -> None: + self._user_data = user_data + + def get_user_data(self) -> Any: + return self._user_data + + def cancel(self) -> None: + if self._is_finished: + raise ValueError('Session already finished') + + self._cancellable.cancel() + + def set_request_body_from_path(self, content_type: str, path: Path) -> None: + if not path.exists(): + raise ValueError('%s does not exist' % path) + + if not path.is_file(): + raise ValueError('%s is not a file' % path) + + self._request_body_file = Gio.File.new_for_path(str(path)) + self._request_content_type = content_type + self._request_content_length = path.stat().st_size + + def set_request_body(self, content_type: str, data: bytes) -> None: + self._request_body_data = data + self._request_content_type = content_type + self._request_content_length = len(data) + + def set_response_body_from_path(self, path: Path) -> None: + self._response_body_file = Gio.File.new_for_path(str(path)) + + def connect(self, + signal_name: str, + callback: Any, + *args: Any + ) -> None: + + if signal_name == 'response-progress': + self._emit_response_progress = True + + if signal_name == 'request-progress': + self._emit_request_progress = True + + if signal_name in GObject.signal_list_names(HTTPRequest): + GObject.GObject.connect(self, signal_name, callback, *args) + return + + user_data = (callback, args) + + self._message.connect(signal_name, + self._on_connect_callback, + user_data) + + def _on_connect_callback(self, _message: Soup.Message, *args: Any) -> None: + signal_args = args[:-1] + callback, user_data = args[-1] + callback(self, *signal_args, *user_data) + + def send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None, + callback: Optional[Callable[[HTTPRequest], Any]] = None + ) -> None: + + if callback: + self.connect('finished', callback) + self._send(method, uri_string, timeout) + + def _send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None + ) -> None: + + if self._is_finished: + raise ValueError('Session already finished') + + self._message.set_method(method) + + if self._response_body_file is not None: + self._output_stream = self._response_body_file.replace( + None, + False, + Gio.FileCreateFlags.REPLACE_DESTINATION, + self._cancellable) + + uri = GLib.Uri.parse(uri_string, GLib.UriFlags.HAS_PASSWORD) + self._message.set_uri(uri) + + if self._request_body_data is not None: + self._message.set_request_body_from_bytes( + self._request_content_type, + GLib.Bytes.new(self._request_body_data) + ) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + if self._request_body_file is not None: + request_input_stream = self._request_body_file.read( + self._cancellable) + self._message.set_request_body(self._request_content_type, + request_input_stream, + self._request_content_length) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + self._message.connect('finished', self._on_finished) + + soup_session = self._session.get_soup_session() + soup_session.send_async(self._message, + GLib.PRIORITY_DEFAULT, + self._cancellable, + self._on_response) + + if timeout is not None: + self._timeout_id = GLib.timeout_add_seconds( + timeout, self._on_timeout) + + self._log.info('Request sent') + + def _on_request_body_progress(self, + _message: Soup.Message, + chunk: int) -> None: + + self._sent_size += chunk + self.emit('request-progress', + self._sent_size / self._request_content_length) + + def _on_timeout(self) -> None: + self._timeout_reached = True + self.cancel() + + def _on_response(self, + session: Soup.Session, + result: Gio.AsyncResult + ) -> None: + + self._log.info('Request response received') + try: + self._input_stream = session.send_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._set_failed(HTTPRequestError.CANCELLED) + else: + self._set_failed(HTTPRequestError.UNKNOWN) + return + + if self._message.get_status() not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + headers = self.get_response_headers() + self._response_content_length = headers.get_content_length() + self._response_content_type, _params = headers.get_content_type() + if self._response_content_length == 0: + self._set_failed(HTTPRequestError.MISSING_CONTENT_LENGTH) + return + + self._log.info('Start downloading response body') + self.emit('starting-response-body') + + self._read_async() + + def _read_async(self) -> None: + self._input_stream.read_bytes_async(CHUNK_SIZE, + GLib.PRIORITY_LOW, + self._cancellable, + self._on_bytes_read_result) + + def _on_bytes_read_result(self, + input_stream: Gio.InputStream, + result: Gio.AsyncResult) -> None: + try: + data = input_stream.read_bytes_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + + return + + bytes_ = data.get_data() + if not bytes_: + self._finish_read() + return + + length = len(bytes_) + self._received_size += length + if self._received_size > self._response_content_length: + self._finish_read(HTTPRequestError.CONTENT_OVERFLOW) + return + + if self._output_stream is None: + self._response_body_data += bytes_ + + else: + try: + self._output_stream.write_all(bytes_, self._cancellable) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + return + + self._read_async() + + if (self._emit_response_progress and + self._message.get_method() == 'GET'): + self.emit('response-progress', + self._received_size / self._response_content_length) + + + def _finish_read(self, error: Optional[HTTPRequestError] = None) -> None: + self._log.info('Finished reading') + if error is None: + self._close_all_streams() + return + + self._set_failed(error) + + def _on_finished(self, _message: Soup.Message) -> None: + self._set_finished() + + def _set_finished(self) -> None: + status = self._message.get_status() + if status == Soup.Status.NONE: + # Message has not been sent, can happen when we cancel the message + # before it is sent. The finished signal triggers before the + # cancelled exception. + return + + if self._is_finished: + return + + headers = self._message.get_response_headers() + content_length = headers.get_content_length() + if self._received_size != content_length: + self._set_failed(HTTPRequestError.INCOMPLETE) + return + + if status not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + self._set_complete() + + def _set_failed(self, error: HTTPRequestError) -> None: + self._is_finished = True + if self._timeout_reached: + self._timeout_id = None + self._error = HTTPRequestError.TIMEOUT + else: + self._error = error + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _set_complete(self) -> None: + self._is_finished = True + self._is_complete = True + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _close_all_streams(self) -> None: + if self._input_stream is not None: + if not self._input_stream.is_closed(): + self._input_stream.close(None) + + if self._output_stream is not None: + if not self._output_stream.is_closed(): + self._output_stream.close(None) + + def _cleanup(self) -> None: + self._log.info('Run cleanup') + + self._response_body_data = b'' + self._message.run_dispose() + + del self._cancellable + del self._session + del self._input_stream + del self._output_stream + del self._user_data + + if self._timeout_id is not None: + GLib.source_remove(self._timeout_id) + + self.emit('destroy') + self.run_dispose() + + def __repr__(self) -> str: + return f'Request({id(self)})' ===================================== nbxmpp/modules/muclumbus.py ===================================== @@ -17,8 +17,6 @@ import json -from gi.repository import Soup - from nbxmpp.namespaces import Namespace from nbxmpp.protocol import Node from nbxmpp.protocol import Iq @@ -44,15 +42,6 @@ class Muclumbus(BaseModule): self._client = client self.handlers = [] - self._proxy_resolver = None - self._soup_session = Soup.Session() - - def set_proxy(self, proxy): - if proxy is None: - return - self._proxy_resolver = proxy.get_resolver() - self._soup_session.props.proxy_resolver = self._proxy_resolver - @iq_request_task def request_parameters(self, jid): task = yield @@ -141,24 +130,25 @@ class Muclumbus(BaseModule): if after is not None: search['after'] = after - message = Soup.Message.new('POST', uri) - message.set_request('application/json', - Soup.MemoryUse.COPY, - json.dumps(search).encode()) + body = json.dumps(search).encode() - response_message = yield message + session = self._client.http_session + request = session.create_request() + request.set_request_body('application/json', body) + request.send('POST', uri) - soup_body = response_message.get_property('response-body') + request = yield request - if response_message.status_code != 200: - self._log.warning(soup_body.data) + if not request.is_complete(): + self._log.warning(request.get_error_string()) yield MuclumbusResult(first=None, last=None, max=None, end=True, items=[]) - response = json.loads(soup_body.data) + response_body = request.get_data() + response = json.loads(response_body) result = response['result'] items = result.get('items') ===================================== nbxmpp/structs.py ===================================== @@ -30,8 +30,8 @@ from dataclasses import dataclass from dataclasses import field from datetime import datetime -from gi.repository import Soup from gi.repository import Gio +from gi.repository import GLib from nbxmpp.simplexml import Node from nbxmpp.namespaces import Namespace @@ -708,8 +708,9 @@ class ProxyData(NamedTuple): def get_uri(self) -> str: if self.username is not None: - user_pass = Soup.uri_encode('%s:%s' % (self.username, - self.password)) + username = GLib.uri_escape_string(self.username, None, False) + password = GLib.uri_escape_string(self.password, None, False) + user_pass = f'{username}:{password}' return '%s://%s@%s' % (self.type, user_pass, self.host) ===================================== nbxmpp/task.py ===================================== @@ -27,14 +27,14 @@ import logging from enum import IntEnum from functools import wraps -from gi.repository import Soup from gi.repository import GLib -from nbxmpp.modules.base import BaseModule from nbxmpp.simplexml import Node from nbxmpp.errors import is_error from nbxmpp.errors import CancelledError from nbxmpp.errors import TimeoutStanzaError +from nbxmpp.http import HTTPRequest +from nbxmpp.modules.base import BaseModule from nbxmpp.modules.util import make_func_arguments_string @@ -109,8 +109,7 @@ def http_request_task(func): @wraps(func) def func_wrapper(self, *args, callback=None, user_data=None, **kwargs): task = HTTPRequestTask(func(self, *args, **kwargs), - self._log, - self._soup_session) + self._log) return _setup_task(task, self._client, callback, user_data) return func_wrapper @@ -251,7 +250,7 @@ class Task: def _invoke_callbacks(self): for callback in self._done_callbacks: - if isinstance(callback, weakref.WeakMethod): + if isinstance(callback, (weakref.WeakMethod, weakref.ref)): callback = callback() if callback is None: return @@ -377,19 +376,21 @@ class HTTPRequestTask(Task): ''' - _process_types = (Soup.Message,) + _process_types = (HTTPRequest,) - def __init__(self, gen, logger, session): + def __init__(self, gen, logger): super().__init__(gen, logger) - self._session = session - def _run_async(self, message): + def _run_async(self, request: HTTPRequest): # pylint: disable=arguments-renamed - self._session.queue_message(message, self._async_finished, None) + request.connect('finished', self._async_finished) + + def _async_finished(self, request: HTTPRequest): + + if self._state == TaskState.CANCELLED: + return - def _async_finished(self, _session, message, _user_data): - if self._state != TaskState.CANCELLED: - self._next_step(message) + self._next_step(request) def _finalize(self): self._session = None ===================================== nbxmpp/websocket.py ===================================== @@ -22,7 +22,6 @@ from gi.repository import GLib from gi.repository import Gio from nbxmpp.const import TCPState -from nbxmpp.const import ConnectionType from nbxmpp.util import get_websocket_close_string from nbxmpp.util import convert_tls_error_flags from nbxmpp.connection import Connection @@ -35,11 +34,10 @@ class WebsocketConnection(Connection): Connection.__init__(self, *args, **kwargs) self._session = Soup.Session() - self._session.props.ssl_strict = False if self._log.getEffectiveLevel() == logging.INFO: self._session.add_feature( - Soup.Logger.new(Soup.LoggerLogLevel.BODY, -1)) + Soup.Logger.new(Soup.LoggerLogLevel.BODY)) self._websocket = None self._cancellable = Gio.Cancellable() @@ -53,11 +51,12 @@ class WebsocketConnection(Connection): self.state = TCPState.CONNECTING message = Soup.Message.new('GET', self._address.uri) - message.connect('starting', self._check_certificate) + message.connect('accept-certificate', self._check_certificate) message.set_flags(Soup.MessageFlags.NO_REDIRECT) self._session.websocket_connect_async(message, None, ['xmpp'], + GLib.PRIORITY_DEFAULT, self._cancellable, self._on_connect, None) @@ -91,19 +90,20 @@ class WebsocketConnection(Connection): # Soup.Session does this automatically raise NotImplementedError - def _check_certificate(self, message): - https_used, certificate, errors = message.get_https_status() - if not https_used and self._address.type == ConnectionType.PLAIN: - return + def _check_certificate(self, + _message, + certificate, + errors): self._peer_certificate = certificate self._peer_certificate_errors = convert_tls_error_flags(errors) if self._accept_certificate(): - return + return True self.notify('bad-certificate') self._cancellable.cancel() + return False def _on_websocket_message(self, _websocket, _type, message): data = message.get_data().decode() ===================================== test/unit/test_http.py ===================================== @@ -0,0 +1,247 @@ +from time import time +import tempfile +import unittest +from unittest.mock import call +from unittest.mock import Mock +from unittest.mock import patch +import os +from pathlib import Path + +from gi.repository import GLib +from gi.repository import Soup + +from nbxmpp.const import HTTPRequestError +from nbxmpp.http import HTTPSession + + +SMALL_FILE_URL = 'https://gajim.org/downloads/ci/unittest_small_file' # 200 KB +BIG_FILE_URL = 'https://gajim.org/downloads/ci/unittest_big_file' # 7 MB +LARGE_FILE_URL = 'https://gajim.org/downloads/ci/unittest_large_file' # 80 MB +NO_FILE_URL = 'https://gajim.org/downloads/ci/no-file' + + +# import logging +# consoleloghandler = logging.StreamHandler() +# log = logging.getLogger('nbxmpp.http') +# log.setLevel(logging.DEBUG) +# log.addHandler(consoleloghandler) + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class HTTP(unittest.TestCase): + + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def test_download_to_file(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + + temp_dir = tempfile.gettempdir() + request.set_response_body_from_path(Path(temp_dir) / 'file') + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_in_memory_download(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_before_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + GLib.timeout_add(10, request.cancel) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.progress.assert_not_called() + callback_mock.starting.assert_not_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_after_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + def _on_progress(req, progress): + self.assertIsInstance(progress, float) + req.cancel() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('response-progress', _on_progress) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', LARGE_FILE_URL, timeout=10) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_failed_404(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('finished', callback_mock.finished) + request.connect('response-progress', callback_mock.progress) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', NO_FILE_URL, timeout=5) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.STATUS_NOT_OK) + self.assertEqual(request.get_status(), Soup.Status.NOT_FOUND) + + callback_mock.progress.assert_not_called() + callback_mock.finished.assert_called() + + def test_cancel_with_timeout(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', LARGE_FILE_URL, timeout=1) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertFalse(request.is_complete()) + self.assertEqual(request.get_error(), HTTPRequestError.TIMEOUT) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_signal_propagation(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting', callback_mock.starting, 1, 2, 3) + request.connect('got-body', callback_mock.got_body) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertTrue(request.is_complete()) + + callback_mock.starting.assert_called_with(request, 1, 2, 3) + callback_mock.got_body.assert_called_with(request) + + def test_parallel_download(self): + + mainloop = GLib.MainLoop() + + request1 = HTTPSession().create_request() + request1.send('GET', SMALL_FILE_URL, timeout=5) + + request2 = HTTPSession().create_request() + request2.send('GET', SMALL_FILE_URL, timeout=5) + + request3 = HTTPSession().create_request() + request3.send('GET', SMALL_FILE_URL, timeout=5) + + request4 = HTTPSession().create_request() + request4.send('GET', SMALL_FILE_URL, timeout=5) + + GLib.timeout_add_seconds(5, mainloop.quit) + + mainloop.run() + + self.assertTrue(request1.is_finished()) + self.assertTrue(request1.is_complete()) + + self.assertTrue(request2.is_finished()) + self.assertTrue(request2.is_complete()) + + self.assertTrue(request3.is_finished()) + self.assertTrue(request3.is_complete()) + + self.assertTrue(request4.is_finished()) + self.assertTrue(request4.is_complete()) + + +if __name__ == '__main__': + unittest.main() ===================================== test/unit/test_muclumbus.py ===================================== @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Any + +from gi.repository import GLib + +import os +import unittest +from unittest.mock import Mock +from nbxmpp.client import Client +from nbxmpp.http import HTTPSession +from nbxmpp.structs import MuclumbusResult +from nbxmpp.modules.muclumbus import Muclumbus + +from nbxmpp.sasl import SCRAM_SHA_1 +from nbxmpp.util import b64encode + +# Test vector from https://wiki.xmpp.org/web/SASL_and_SCRAM-SHA-1 + +API_URL = 'https://search.jabber.network/api/1.0/search' + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class TestMuclumbus(unittest.TestCase): + def setUp(self): + self._client = Client() + self._client.set_http_session(HTTPSession()) + self._module = Muclumbus(self._client) + + def test_http_request(self): + + mainloop = GLib.MainLoop() + + def _result(task: Any) -> None: + result = task.finish() + assert isinstance(result, MuclumbusResult) + assert len(result.items) > 0 + mainloop.quit() + + self._module.set_http_search(API_URL, 'gajim', callback=_result) + + mainloop.run() + +if __name__ == '__main__': + unittest.main() View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/0da8888db96f23a7329802af… -- View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/0da8888db96f23a7329802af… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/python-nbxmpp] Deleted branch soup3
by Philipp Hörist (＠lovetox) 28 Dec '22

28 Dec '22

Philipp Hörist deleted branch soup3 at gajim / python-nbxmpp -- You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][soup3] change: Port to libsoup3
by Philipp Hörist (＠lovetox) 28 Dec '22

28 Dec '22

Philipp Hörist pushed to branch soup3 at gajim / gajim Commits: 385aa245 by Philipp Hörist at 2022-12-28T11:29:49+01:00 change: Port to libsoup3 - - - - - 15 changed files: - .gitlab-ci.yml - README.md - gajim/common/application.py - gajim/common/client.py - gajim/common/filetransfer.py - gajim/common/helpers.py - gajim/common/modules/httpupload.py - gajim/common/preview.py - + gajim/common/util/http.py - gajim/gajim.py - gajim/gtk/conversation/rows/file_transfer.py - gajim/gtk/exception.py - gajim/plugins/repository.py - win/_base.sh - win/dev_env.sh Changes: ===================================== .gitlab-ci.yml ===================================== @@ -26,7 +26,7 @@ test-pyright: - "**/*.py" script: - pip3 install git+https://dev.gajim.org/gajim/python-nbxmpp.git - - PYGOBJECT_STUB_CONFIG=Gtk3,Gdk3,Soup2 pip3 install git+https://github.com/pygobject/pygobject-stubs.git + - PYGOBJECT_STUB_CONFIG=Gtk3,Gdk3 pip3 install git+https://github.com/pygobject/pygobject-stubs.git - npm install pyright - python3 .ci/link-gtk.py - $(npm bin)/pyright --version ===================================== README.md ===================================== @@ -18,7 +18,6 @@ ### Runtime Requirements - [GLib](https://gitlab.com/gnome/glib) (>=2.60.0) - [GtkSourceView](https://gitlab.gnome.org/GNOME/gtksourceview) - [Pango](https://gitlab.gnome.org/GNOME/pango) (>=1.50.0) -- [libsoup](https://gitlab.gnome.org/GNOME/libsoup/) - [sqlite](https://www.sqlite.org/) (>=3.33.0) ### Optional Runtime Requirements ===================================== gajim/common/application.py ===================================== @@ -29,9 +29,9 @@ from gi.repository import Gio from gi.repository import GLib -from gi.repository import Soup from nbxmpp.const import ConnectionType from nbxmpp.const import ConnectionProtocol +from nbxmpp.http import HTTPRequest from gajim.common import app from gajim.common import ged @@ -48,7 +48,6 @@ from gajim.common.events import GajimUpdateAvailable from gajim.common.events import SignedIn from gajim.common.client import Client -from gajim.common.helpers import make_http_request from gajim.common.helpers import get_random_string from gajim.common.helpers import get_global_show from gajim.common.helpers import from_one_line @@ -60,6 +59,7 @@ from gajim.common.storage.cache import CacheStorage from gajim.common.storage.draft import DraftStorage from gajim.common.storage.archive import MessageArchiveStorage +from gajim.common.util.http import create_http_request from gajim.plugins.pluginmanager import PluginManager from gajim.plugins.repository import PluginRepository @@ -310,22 +310,21 @@ def _check_for_updates(self) -> None: def check_for_gajim_updates(self) -> None: self._log.info('Checking for Gajim updates') - make_http_request('https://gajim.org/current-version.json', - self._on_update_response) + request = create_http_request() + request.send('GET', 'https://gajim.org/current-version.json', + callback=self._on_update_response) - def _on_update_response(self, - _session: Soup.Session, - message: Soup.Message) -> None: + def _on_update_response(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + self._log.warning('Error while performing gajim update check: %s', + request.get_error(), request.get_status()) + return now = datetime.now() app.settings.set('last_update_check', now.strftime('%Y-%m-%d %H:%M')) - response_body = message.props.response_body - if response_body is None or not response_body.data: - self._log.warning('Could not reach gajim.org for update check') - return - - data = json.loads(response_body.data) + data = json.loads(request.get_data()) latest_version = data['current_version'] if V(latest_version) > V(app.version): ===================================== gajim/common/client.py ===================================== @@ -55,6 +55,7 @@ from gajim.common.idle import IdleMonitorManager from gajim.common.i18n import _ from gajim.common.structs import OutgoingMessage +from gajim.common.util.http import create_http_session from gajim.gui.util import open_window @@ -175,6 +176,8 @@ def _create_client(self) -> None: self._client.set_password(self.password) + self._client.set_http_session(create_http_session()) + self._client.subscribe('resume-failed', self._on_resume_failed) self._client.subscribe('resume-successful', self._on_resume_successful) self._client.subscribe('disconnected', self._on_disconnected) ===================================== gajim/common/filetransfer.py ===================================== @@ -27,8 +27,7 @@ def __init__(self, account: str) -> None: self._account = account - self._seen: int = 0 - self.size: int = 0 + self._progress = 0 self._state = FTState.INIT self._error_text: str = '' @@ -42,16 +41,6 @@ def account(self) -> str: def state(self) -> FTState: return self._state - @property - def seen(self) -> int: - return self._seen - - @property - def is_complete(self) -> bool: - if self.size == 0: - return False - return self._seen >= self.size - @property def filename(self) -> str: raise NotImplementedError @@ -64,6 +53,13 @@ def error_text(self) -> str: def error_domain(self) -> Optional[str]: return self._error_domain + def get_progress(self) -> float: + return self._progress + + def set_progress(self, progress: float) -> None: + self._progress = progress + self.update_progress() + def get_state_description(self) -> str: if self._state is None: return '' ===================================== gajim/common/helpers.py ===================================== @@ -81,7 +81,6 @@ from gi.repository import Gio from gi.repository import GLib from gi.repository import GObject -from gi.repository import Soup import precis_i18n.codec # noqa: F401 @@ -1546,24 +1545,3 @@ def make_path_from_jid(base_path: Path, jid: JID) -> Path: if jid.resource is not None: return path / jid.resource[:30] return path - - -def make_http_request(uri: str, - callback: Any, - user_data: Optional[Any] = None) -> None: - - proxy = determine_proxy() - if proxy is None: - resolver = None - - else: - resolver = proxy.get_resolver() - - session = Soup.Session() - session.props.proxy_resolver = resolver - session.props.user_agent = f'Gajim {app.version}' - message = Soup.Message.new('GET', uri) - if user_data is None: - session.queue_message(message, callback) - else: - session.queue_message(message, callback, user_data) ===================================== gajim/common/modules/httpupload.py ===================================== @@ -20,12 +20,13 @@ from typing import Callable from typing import Optional -import os -import io -from urllib.parse import urlparse -import mimetypes from collections import defaultdict +import functools from pathlib import Path +from urllib.parse import urlparse +import mimetypes +import os +import tempfile from nbxmpp.errors import StanzaError from nbxmpp.errors import MalformedStanzaError @@ -36,23 +37,25 @@ from nbxmpp.structs import HTTPUploadData from nbxmpp.task import Task from nbxmpp.util import convert_tls_error_flags +from nbxmpp.http import HTTPRequest +from nbxmpp.const import HTTPRequestError from gi.repository import Gio from gi.repository import GLib -from gi.repository import Soup from gajim.common import app from gajim.common import types from gajim.common.events import HTTPUploadError from gajim.common.events import HTTPUploadStarted from gajim.common.i18n import _ +from gajim.common.helpers import get_random_string from gajim.common.helpers import get_tls_error_phrases -from gajim.common.helpers import get_account_proxy from gajim.common.const import FTState from gajim.common.filetransfer import FileTransfer from gajim.common.modules.base import BaseModule from gajim.common.exceptions import FileError from gajim.common.structs import OutgoingMessage +from gajim.common.util.http import create_http_request class HTTPUpload(BaseModule): @@ -67,24 +70,11 @@ def __init__(self, con: types.Client) -> None: self.httpupload_namespace: Optional[str] = None self.max_file_size: Optional[float] = None # max file size in bytes - self._proxy_resolver: Optional[Gio.SimpleProxyResolver] = None - self._queued_messages: dict[int, Soup.Message] = {} - self._session = Soup.Session() - self._session.props.ssl_strict = False - self._session.props.user_agent = f'Gajim {app.version}' + self._requests_in_progress: dict[int, HTTPRequest] = {} self._running_transfers: dict[ tuple[str, JID], set[HTTPFileTransfer]] = defaultdict(set) - def _set_proxy_if_available(self) -> None: - proxy = get_account_proxy(self._account) - if proxy is None: - self._proxy_resolver = None - self._session.props.proxy_resolver = None - else: - self._proxy_resolver = proxy.get_resolver() - self._session.props.proxy_resolver = self._proxy_resolver - def pass_disco(self, info: DiscoInfo) -> None: if not info.has_httpupload: return @@ -215,11 +205,11 @@ def _on_cancel_upload(self, key = (transfer.account, transfer.contact.jid) self._running_transfers[key].discard(transfer) - message = self._queued_messages.get(id(transfer)) - if message is None: + request = self._requests_in_progress.get(id(transfer)) + if request is None: return - self._session.cancel_message(message, Soup.Status.CANCELLED) + request.cancel() def _start_transfer(self, transfer: HTTPFileTransfer) -> None: if transfer.encryption is not None and not transfer.is_encrypted: @@ -283,106 +273,61 @@ def _upload_file(self, transfer: HTTPFileTransfer) -> None: transfer.set_started() assert transfer.put_uri is not None - message = Soup.Message.new('PUT', transfer.put_uri) - message.connect('starting', self._check_certificate, transfer) - - # Set CAN_REBUILD so chunks get discarded after they have been - # written to the network - message.set_flags(Soup.MessageFlags.CAN_REBUILD | - Soup.MessageFlags.NO_REDIRECT) + request = create_http_request(self._account) + request.set_user_data(transfer) + request.connect('accept-certificate', self._accept_certificate) + request.connect('request-progress', self._on_request_progress) - assert message.props.request_body is not None - message.props.request_body.set_accumulate(False) + request.set_request_body_from_path(transfer.mime, transfer.payload_path) - assert message.props.request_headers is not None - message.props.request_headers.set_content_type(transfer.mime, None) - message.props.request_headers.set_content_length(transfer.size) for name, value in transfer.headers.items(): - message.props.request_headers.append(name, value) - - message.connect('wrote-headers', self._on_wrote_headers, transfer) - message.connect('wrote-chunk', self._on_wrote_chunk, transfer) - - self._queued_messages[id(transfer)] = message - self._set_proxy_if_available() - self._session.queue_message(message, self._on_finish, transfer) - - def _check_certificate(self, - message: Soup.Message, - transfer: HTTPFileTransfer - ) -> None: - https_used, tls_certificate, tls_errors = message.get_https_status() - if not https_used: - self._log.warning('HTTPS was not used for upload') - transfer.set_error('unsecure') - self._session.cancel_message(message, Soup.Status.CANCELLED) - return + request.get_request_headers().append(name, value) - tls_error_set = convert_tls_error_flags(tls_errors) - if app.cert_store.verify(tls_certificate, tls_error_set): - return + request.send('PUT', transfer.put_uri, callback=self._on_finish) + + self._requests_in_progress[id(transfer)] = request + + def _accept_certificate(self, + request: HTTPRequest, + _certificate: Gio.TlsCertificate, + certificate_errors: Gio.TlsCertificateFlags, + ) -> bool: - phrases = get_tls_error_phrases(tls_error_set) + transfer = request.get_user_data() + phrases = get_tls_error_phrases( + convert_tls_error_flags(certificate_errors)) self._log.warning( - 'TLS verification failed: %s (0x%02x)', phrases, tls_errors) + 'TLS verification failed: %s (0x%02x)', phrases, certificate_errors) + transfer.set_error('tls-verification-failed', phrases[0]) - self._session.cancel_message(message, Soup.Status.CANCELLED) + return False - def _on_finish(self, - _session: Soup.Session, - message: Soup.Message, - transfer: HTTPFileTransfer - ) -> None: + def _on_finish(self, request: HTTPRequest) -> None: + transfer = request.get_user_data() - self._queued_messages.pop(id(transfer), None) + self._requests_in_progress.pop(id(transfer), None) key = (transfer.account, transfer.contact.jid) self._running_transfers[key].discard(transfer) - if message.props.status_code == Soup.Status.CANCELLED: - self._log.info('Upload cancelled') - return - - if message.props.status_code in (Soup.Status.OK, Soup.Status.CREATED): - self._log.info('Upload completed successfully') - transfer.set_finished() - - else: - phrase = Soup.Status.get_phrase(message.props.status_code) - self._log.error('Got unexpected http upload response code: %s', - phrase) - transfer.set_error('http-response', phrase) - - def _on_wrote_chunk(self, - message: Soup.Message, - transfer: HTTPFileTransfer - ) -> None: - transfer.update_progress() - if transfer.is_complete: - assert message.props.request_body is not None - message.props.request_body.complete() + if not request.is_complete(): + error = request.get_error_string() + if request.get_error() == HTTPRequestError.CANCELLED: + self._log.info('Upload cancelled') + else: + transfer.set_error('http-response', error) return - bytes_ = transfer.get_chunk() - assert bytes_ is not None - self._session.pause_message(message) - GLib.idle_add(self._append, message, bytes_) + transfer.set_finished() + self._log.info('Upload completed successfully') - def _append(self, message: Soup.Message, bytes_: bytes) -> None: - if message.props.status_code == Soup.Status.CANCELLED: - return - self._session.unpause_message(message) - assert message.props.request_body is not None - message.props.request_body.append(bytes_) + def _on_request_progress(self, + request: HTTPRequest, + progress: float + ) -> None: - @staticmethod - def _on_wrote_headers(message: Soup.Message, - transfer: HTTPFileTransfer - ) -> None: - bytes_ = transfer.get_chunk() - assert bytes_ is not None - assert message.props.request_body is not None - message.props.request_body.append(bytes_) + transfer = request.get_user_data() + transfer.set_progress(progress) class HTTPFileTransfer(FileTransfer): @@ -410,23 +355,32 @@ def __init__(self, FileTransfer.__init__(self, account) - self._path = path + self._path = Path(path) self._encryption = encryption self._groupchat = groupchat self._contact = contact self._mime = mime - self.size = os.stat(path).st_size self.put_uri: Optional[str] = None self.get_uri: Optional[str] = None self._uri_transform_func: Optional[Callable[[str], str]] = None - self._stream = None self._data: Optional[bytes] = None self._headers: dict[str, str] = {} self._is_encrypted = False + @property + def size(self) -> int: + if self._encryption is not None and not self._is_encrypted: + raise ValueError('File size unknown at this point') + return self.payload_path.stat().st_size + + @size.setter + def size(self, size: int) -> None: + # Backwards compatibility with plugins + pass + @property def mime(self) -> str: return self._mime @@ -448,7 +402,13 @@ def headers(self) -> dict[str, str]: return self._headers @property - def path(self) -> str: + def path(self) -> Path: + return self._path + + @property + def payload_path(self) -> Path: + if self._encryption is not None: + return self.get_temp_path() return self._path @property @@ -465,47 +425,30 @@ def set_uri_transform_func(self, func: Callable[[str], str]) -> None: @property def filename(self) -> str: - return os.path.basename(self._path) + return self._path.name + + @staticmethod + @functools.cache + def get_temp_path() -> Path: + tempdir = tempfile.gettempdir() + return Path(tempdir) / get_random_string(16) def set_error(self, domain: str, text: str = '') -> None: if not text: text = self._errors[domain] - self._close() super().set_error(domain, text) def set_finished(self) -> None: - self._close() super().set_finished() def set_encrypted_data(self, data: bytes) -> None: - self._data = data + temp_path = self.get_temp_path() + temp_path.write_bytes(data) self._is_encrypted = True - def _close(self) -> None: - if self._stream is not None: - self._stream.close() - - def get_chunk(self) -> Optional[bytes]: - if self._stream is None: - if self._encryption is None: - self._stream = open(self._path, 'rb') # pylint: disable=consider-using-with # noqa: E501 - else: - self._stream = io.BytesIO(self._data) - - data = self._stream.read(16384) - if not data: - self._close() - return None - self._seen += len(data) - if self.is_complete: - self._close() - return data - def get_data(self) -> bytes: - with open(self._path, 'rb') as file: - data = file.read() - return data + return self._path.read_bytes() def process_result(self, result: HTTPUploadData) -> None: self.put_uri = result.put_uri ===================================== gajim/common/preview.py ===================================== @@ -14,12 +14,14 @@ from __future__ import annotations -from dataclasses import dataclass, field from typing import Any +from typing import cast from typing import Callable from typing import Optional import logging +from dataclasses import dataclass +from dataclasses import field import os import re from pathlib import Path @@ -29,7 +31,8 @@ from gi.repository import Gio from gi.repository import GLib -from gi.repository import Soup +from nbxmpp.const import HTTPRequestError +from nbxmpp.http import HTTPRequest from nbxmpp.util import convert_tls_error_flags from gajim.common import app @@ -40,7 +43,6 @@ from gajim.common.helpers import load_file_async from gajim.common.helpers import write_file_async from gajim.common.helpers import get_tls_error_phrases -from gajim.common.helpers import get_account_proxy from gajim.common.i18n import _ from gajim.common.preview_helpers import aes_decrypt from gajim.common.preview_helpers import filename_from_uri @@ -52,6 +54,7 @@ from gajim.common.preview_helpers import guess_mime_type from gajim.common.preview_helpers import pixbuf_from_data from gajim.common.types import GdkPixbufType +from gajim.common.util.http import create_http_request log = logging.getLogger('gajim.c.preview') @@ -104,12 +107,11 @@ def __init__(self, self.thumbnail: Optional[bytes] = None self.mime_type: str = '' self.file_size: int = 0 - self._received_size: int = 0 self.download_in_progress = False self.info_message: Optional[str] = None - self._soup_message: Optional[Soup.Message] = None + self._request = None self.key: Optional[bytes] = None self.iv: Optional[bytes] = None @@ -154,8 +156,8 @@ def filename(self) -> str: return self._filename @property - def soup_message(self) -> Soup.Message: - return self._soup_message + def request(self) -> HTTPRequest: + return self._request @property def request_uri(self) -> Optional[str]: @@ -193,29 +195,17 @@ def create_thumbnail(self, data: bytes) -> bool: return False return True - def reset_received_size(self) -> None: - self._received_size = 0 - def update_widget(self, data: Optional[GdkPixbufType] = None) -> None: self._widget.update(self, data) - def update_progress(self, size: int, message: Soup.Message) -> None: + def update_progress(self, progress: float, request: HTTPRequest) -> None: self.download_in_progress = True - self._soup_message = message - self._received_size += size - if self.file_size == 0 or self._received_size == 0: - return - - progress = self._received_size / self.file_size + self._request = request self._widget.update_progress(self, progress) class PreviewManager: def __init__(self) -> None: - self._sessions: dict[ - str, - tuple[Soup.Session, Optional[Gio.SimpleProxyResolver]]] = {} - self._orig_dir = Path(configpaths.get('MY_DATA')) / 'downloads' self._thumb_dir = Path(configpaths.get('MY_CACHE')) / 'downloads.thumb' @@ -268,29 +258,6 @@ def stop_audio_except(self, preview_id: int) -> None: if id_ != preview_id: stop_func() - def _get_session(self, account: str) -> Soup.Session: - if account not in self._sessions: - self._sessions[account] = self._create_session(account) - return self._sessions[account][0] - - @staticmethod - def _create_session(account: str) -> tuple[ - Soup.Session, Optional[Gio.SimpleProxyResolver]]: - session = Soup.Session() - session.add_feature_by_type(Soup.ContentSniffer) - session.props.https_aliases = ['aesgcm'] - session.props.ssl_strict = False - session.props.user_agent = f'Gajim {app.version}' - - proxy = get_account_proxy(account) - if proxy is None: - resolver = None - else: - resolver = proxy.get_resolver() - - session.props.proxy_resolver = resolver - return session, resolver - @staticmethod def _accept_uri(urlparts: ParseResult, uri: str, @@ -476,58 +443,62 @@ def download_content(self, log.info('Download already in progress') return - preview.reset_received_size() if preview.account is None: # History Window can be opened without account context # This means we can not apply proxy settings return log.info('Start downloading: %s', preview.request_uri) - message = Soup.Message.new('GET', preview.request_uri) - message.connect('starting', self._check_certificate, preview) - message.connect( - 'content-sniffed', self._on_content_sniffed, preview, force) - message.connect('got-chunk', self._on_got_chunk, preview) - session = self._get_session(preview.account) - session.queue_message(message, self._on_finished, preview) + request = create_http_request(preview.account, sniffer=True) + request.set_user_data(preview) + request.connect('accept-certificate', self._accept_certificate) + request.connect('content-sniffed', self._on_content_sniffed, force) + request.connect('response-progress', self._on_response_progress) - def _check_certificate(self, - message: Soup.Message, - preview: Preview) -> None: - _https_used, _tls_certificate, tls_errors = message.get_https_status() + request.send('GET', preview.request_uri, callback=self._on_finished) + + def _accept_certificate(self, + request: HTTPRequest, + certificate: Gio.TlsCertificate, + certificate_errors: Gio.TlsCertificateFlags, + ) -> bool: if not app.settings.get('preview_verify_https'): - return + return True - if tls_errors: - phrases = get_tls_error_phrases(convert_tls_error_flags(tls_errors)) - log.warning( - 'TLS verification failed: %s (0x%02x)', phrases, tls_errors) - session = self._get_session(preview.account) - session.cancel_message(message, Soup.Status.CANCELLED) - preview.info_message = _('TLS verification failed: %s') % phrases[0] - preview.update_widget() + phrases = get_tls_error_phrases( + convert_tls_error_flags(certificate_errors)) + log.warning( + 'TLS verification failed: %s (0x%02x)', phrases, certificate_errors) + + preview = cast(Preview, request.get_user_data()) + preview.info_message = _('TLS verification failed: %s') % phrases[0] + preview.update_widget() + return False def _on_content_sniffed(self, - message: Soup.Message, - type_: str, + request: HTTPRequest, + content_type: str, _params: GLib.HashTable, - preview: Preview, - force: bool) -> None: - file_size = message.props.response_headers.get_content_length() - uri = message.props.uri.to_string(False) - session = self._get_session(preview.account) - preview.mime_type = type_ + force: bool + ) -> None: + + file_size = request.get_response_headers().get_content_length() + uri = request.get_uri().to_string() + + + preview = cast(Preview, request.get_user_data()) + preview.mime_type = content_type preview.file_size = file_size - if type_ not in ALLOWED_MIME_TYPES and not force: - log.info('Not an allowed content type: %s, %s', type_, uri) - session.cancel_message(message, Soup.Status.CANCELLED) + if content_type not in ALLOWED_MIME_TYPES and not force: + log.info('Not an allowed content type: %s, %s', content_type, uri) + request.cancel() return if file_size == 0: log.info('File size is unknown (zero) for URL: "%s"', uri) - session.cancel_message(message, Soup.Status.CANCELLED) + request.cancel() return if file_size > int(app.settings.get('preview_max_file_size')): @@ -537,42 +508,37 @@ def _on_content_sniffed(self, if force: preview.info_message = None else: - session.cancel_message(message, Soup.Status.CANCELLED) + request.cancel() preview.info_message = _('Automatic preview disabled ' '(file too big)') preview.update_widget() - def _on_got_chunk(self, - message: Soup.Message, - chunk: Soup.Buffer, - preview: Preview - ) -> None: - preview.update_progress(len(chunk.get_data()), message) + def _on_response_progress(self, + request: HTTPRequest, + progress: float, + ) -> None: - def _on_finished(self, - _session: Soup.Session, - message: Soup.Message, - preview: Preview - ) -> None: + preview = cast(Preview, request.get_user_data()) + preview.update_progress(progress, request) + def _on_finished(self, request: HTTPRequest) -> None: + + preview = cast(Preview, request.get_user_data()) preview.download_in_progress = False - if message.status_code != Soup.Status.OK: - log.warning('Download failed: %s', preview.request_uri) - status_code = Soup.Status.get_phrase(message.status_code) - log.warning(status_code) - preview.reset_received_size() - if message.status_code != 1: - # status_code 1: 'Cancelled' - preview.info_message = _('Download failed (%s)') % status_code + if not request.is_complete(): + error = request.get_error_string() + log.warning('Download failed: %s - %s', preview.request_uri, error) + if request.get_error() != HTTPRequestError.CANCELLED: + preview.info_message = _('Download failed (%s)') % error preview.update_widget() return preview.info_message = None - data = message.props.response_body_data.get_data() - if data is None: + data = request.get_data() + if not data: return if preview.is_aes_encrypted: @@ -649,6 +615,4 @@ def _on_thumb_write_finished(_result: bool, preview.update_widget(data=pixbuf) def cancel_download(self, preview: Preview) -> None: - preview.reset_received_size() - session = self._get_session(preview.account) - session.cancel_message(preview.soup_message, Soup.Status.CANCELLED) + preview.request.cancel() ===================================== gajim/common/util/http.py ===================================== @@ -0,0 +1,48 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see <http://www.gnu.org/licenses/>. + +from typing import Optional + +from nbxmpp.http import HTTPRequest +from nbxmpp.http import HTTPSession + +from gajim.common import app +from gajim.common.helpers import determine_proxy +from gajim.common.helpers import get_account_proxy + + +def create_http_session(account: Optional[str] = None, + sniffer: bool = False + ) -> HTTPSession: + + session = HTTPSession(user_agent=f'Gajim {app.version}', + sniffer=sniffer) + + if account is None: + proxy = determine_proxy() + else: + proxy = get_account_proxy(account) + + if proxy is not None: + session.set_proxy_resolver(proxy.get_resolver()) + + return session + + +def create_http_request(account: Optional[str] = None, + sniffer: bool = False + ) -> HTTPRequest: + + session = create_http_session(account, sniffer) + return session.create_request() ===================================== gajim/gajim.py ===================================== @@ -65,8 +65,7 @@ def _check_required_deps() -> None: 'GtkSource': '4', 'GObject': '2.0', 'Pango': '1.0', - 'PangoCairo': '1.0', - 'Soup': '2.4'}) + 'PangoCairo': '1.0'}) except ValueError as error: sys.exit('Missing dependency: %s' % error) ===================================== gajim/gtk/conversation/rows/file_transfer.py ===================================== @@ -131,7 +131,10 @@ def _on_transfer_progress(self, size_total = GLib.format_size_full(transfer.size, self._units) self._ui.file_size.set_text(size_total) - bytes_sec = int(round(transfer.seen / (time_now - self._start_time), 1)) + progress = transfer.get_progress() + seen = transfer.size * progress + + bytes_sec = int(round(seen / (time_now - self._start_time), 1)) speed = f'{GLib.format_size_full(bytes_sec, self._units)}/s' self._ui.transfer_progress.set_tooltip_text(_('Speed: %s') % speed) @@ -139,9 +142,8 @@ def _on_transfer_progress(self, eta = '∞' else: eta = format_eta(round( - (transfer.size - transfer.seen) / bytes_sec)) + (transfer.size - seen) / bytes_sec)) - progress = float(transfer.seen) / transfer.size self._ui.transfer_progress.set_text( _('%(progress)s %% (%(time)s remaining)') % { 'progress': round(progress * 100), ===================================== gajim/gtk/exception.py ===================================== @@ -20,11 +20,11 @@ from __future__ import annotations from typing import Any -from types import TracebackType import sys import json import traceback +from types import TracebackType import threading import webbrowser import pprint @@ -33,9 +33,9 @@ from gi.repository import Gdk from gi.repository import Gtk -from gi.repository import Soup import nbxmpp +from nbxmpp.http import HTTPRequest import gajim from gajim.common import app @@ -44,8 +44,8 @@ from gajim.common.helpers import get_os_name from gajim.common.helpers import get_os_version from gajim.common.helpers import determine_proxy -from gajim.common.helpers import make_http_request from gajim.common.i18n import _ +from gajim.common.util.http import create_http_request from .builder import get_builder from .util import get_gtk_version @@ -182,24 +182,17 @@ def _request_sentry_endpoint(self) -> None: self._ui.report_spinner.show() self._ui.report_spinner.start() - make_http_request('https://gajim.org/updates.json', - self._on_endpoint_received) + request = create_http_request() + request.send('GET', 'https://gajim.org/updates.json', + callback=self._on_endpoint_received) - def _parse_endpoint(self, message: Soup.Message) -> str: - status = message.props.status_code - response_body = message.props.response_body - - if status != Soup.Status.OK: - error = message.props.reason_phrase + def _parse_endpoint(self, request: HTTPRequest) -> str: + if not request.is_complete(): raise ValueError('Failed to retrieve sentry endpoint: %s %s' % ( - status, error)) - - if response_body is None or not response_body.data: - raise ValueError('Failed to retrieve sentry endpoint, ' - 'no response body') + request.get_status(), request.get_error())) try: - data = json.loads(response_body.data) + data = json.loads(request.get_data()) except Exception as error: raise ValueError('Json parsing error: %s' % error) @@ -209,12 +202,9 @@ def _parse_endpoint(self, message: Soup.Message) -> str: return endpoint - def _on_endpoint_received(self, - _session: Soup.Session, - message: Soup.Message - ) -> None: + def _on_endpoint_received(self, request: HTTPRequest) -> None: try: - endpoint = self._parse_endpoint(message) + endpoint = self._parse_endpoint(request) except ValueError as error: print(error) self._report_with_browser() ===================================== gajim/plugins/repository.py ===================================== @@ -12,7 +12,7 @@ # You should have received a copy of the GNU General Public License # along with Gajim. If not, see <http://www.gnu.org/licenses/>. -from typing import Any +from typing import Any, cast from typing import Iterable from typing import Optional @@ -21,14 +21,14 @@ from io import BytesIO from zipfile import ZipFile -from gi.repository import Soup from gi.repository import GLib +from nbxmpp.http import HTTPRequest from gajim.common import app from gajim.common import configpaths from gajim.common.helpers import Observable -from gajim.common.helpers import make_http_request from gajim.plugins.manifest import PluginManifest +from gajim.common.util.http import create_http_request log = logging.getLogger('gajim.p.repository') @@ -60,28 +60,27 @@ def __init__(self) -> None: self._download_queue: set[PluginManifest] = set() if app.settings.get('plugins_repository_enabled'): - make_http_request('https://gajim.org/updates.json', - self._on_repository_received) + request = create_http_request() + request.send('GET', 'https://gajim.org/updates.json', + callback=self._on_repository_received) @property def available(self): return bool(self._repository_url) - def _on_repository_received(self, - _session: Soup.Session, - message: Soup.Message) -> None: + def _on_repository_received(self, request: HTTPRequest) -> None: - if message.status_code != Soup.Status.OK: - log.warning('Repository retrieval failed') - log.warning(Soup.Status.get_phrase(message.status_code)) + if not request.is_complete(): + log.warning('Repository retrieval failed: %s', + request.get_error_string()) return - assert message.props.response_body_data is not None - data = message.props.response_body_data.get_data() - if data is None: + try: + updates = json.loads(request.get_data()) + except Exception as error: + log.warning('Unable to parse repository information: %s', error) return - updates = json.loads(data.decode()) repository_url = updates.get('plugin_repository') if repository_url is None: log.warning('No plugin repository url found') @@ -148,56 +147,49 @@ def _parse_package_json(self, def _refresh_plugin_index(self, callback: Optional[Any] = None) -> None: log.info('Refresh index') - make_http_request(self._repository_index_url, - self._on_index_received, - callback) - - def _on_index_received(self, - _session: Soup.Session, - message: Soup.Message, - callback: Optional[Any] = None) -> None: - - if message.status_code != Soup.Status.OK: - log.warning('Refresh failed: %s', self._repository_index_url) - log.warning(Soup.Status.get_phrase(message.status_code)) - return + request = create_http_request() + request.set_user_data(callback) + request.send('GET', self._repository_index_url, + callback=self._on_index_received) - assert message.props.response_body_data is not None - data = message.props.response_body_data.get_data() - if data is None: + def _on_index_received(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + log.warning('Refresh failed: %s %s', + self._repository_index_url, + request.get_error_string()) return - package_index = json.loads(data.decode()) + try: + package_index = json.loads(request.get_data()) + except Exception as error: + log.warning('Unable to parse repository index: %s', error) + return self._manifests = self._parse_package_json(package_index) image_path = package_index['metadata']['image_path'] - make_http_request(f'{self._repository_url}/{image_path}', - self._on_images_received) + callback = request.get_user_data() + + request = create_http_request() + request.send('GET', f'{self._repository_url}/{image_path}', + callback=self._on_images_received) log.info('Refresh successful') if callback is not None: callback() - def _on_images_received(self, - _session: Soup.Session, - message: Soup.Message) -> None: + def _on_images_received(self, request: HTTPRequest) -> None: - if message.status_code != Soup.Status.OK: - log.warning('Image download failed') - log.warning(Soup.Status.get_phrase(message.status_code)) - return - - assert message.props.response_body_data is not None - data = message.props.response_body_data.get_data() - if data is None: + if not request.is_complete(): + log.warning('Image download failed: %s', request.get_error_string()) return path = configpaths.get('PLUGINS_IMAGES') try: - zipfile = ZipFile(BytesIO(data)) + zipfile = ZipFile(BytesIO(request.get_data())) zipfile.extractall(path) except Exception as error: log.warning('Failed to extract images: %s', error) @@ -245,27 +237,25 @@ def download_plugins(self, manifests: Iterable[PluginManifest]) -> None: def _download_plugin(self, manifest: PluginManifest) -> None: log.info('Download plugin %s', manifest.short_name) url = manifest.get_remote_url(self._repository_url) - make_http_request(url, self._on_download_plugin_finished, manifest) + request = create_http_request() + request.set_user_data(manifest) + request.send('GET', url, + callback=self._on_download_plugin_finished) - def _on_download_plugin_finished(self, - _session: Soup.Session, - message: Soup.Message, - manifest: PluginManifest) -> None: + def _on_download_plugin_finished(self, request: HTTPRequest) -> None: + manifest = cast(PluginManifest, request.get_user_data()) self._download_queue.remove(manifest) - if message.status_code != Soup.Status.OK: - error = Soup.Status.get_phrase(message.status_code) - log.error('Download failed: %s', - manifest.get_remote_url(self._repository_url)) - log.error(error) + if not request.is_complete(): + error = request.get_error_string() + log.error('Download failed: %s %s', + manifest.get_remote_url(self._repository_url), + error) self.notify('download-failed', manifest, error) return - assert message.props.response_body_data is not None - data = message.props.response_body_data.get_data() - - with ZipFile(BytesIO(data)) as zip_file: + with ZipFile(BytesIO(request.get_data())) as zip_file: zip_file.extractall(self._download_path / manifest.short_name) log.info('Finished downloading %s', manifest.short_name) ===================================== win/_base.sh ===================================== @@ -94,7 +94,7 @@ function install_deps { mingw-w64-"${ARCH}"-goocanvas \ mingw-w64-"${ARCH}"-gspell \ mingw-w64-"${ARCH}"-hunspell \ - mingw-w64-"${ARCH}"-libsoup \ + mingw-w64-"${ARCH}"-libsoup3 \ mingw-w64-"${ARCH}"-python-setuptools \ mingw-w64-"${ARCH}"-python-pillow \ mingw-w64-"${ARCH}"-python-setuptools-scm \ ===================================== win/dev_env.sh ===================================== @@ -22,7 +22,7 @@ function main { mingw-w64-x86_64-goocanvas \ mingw-w64-x86_64-gspell \ mingw-w64-x86_64-hunspell \ - mingw-w64-x86_64-libsoup \ + mingw-w64-x86_64-libsoup3 \ PIP_REQUIREMENTS="\ git+https://dev.gajim.org/gajim/python-nbxmpp.git View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/385aa245462f5fb15fe68ae1a6dfe3ae… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/385aa245462f5fb15fe68ae1a6dfe3ae… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/python-nbxmpp][soup3] change: Port to libsoup3
by Philipp Hörist (＠lovetox) 28 Dec '22

28 Dec '22

Philipp Hörist pushed to branch soup3 at gajim / python-nbxmpp Commits: 0da8888d by Philipp Hörist at 2022-12-28T10:15:01+01:00 change: Port to libsoup3 - Port code to libsoup3 - Wrap Soup.Session and Soup.Message for easier usage - - - - - 15 changed files: - .gitlab-ci.yml - README.md - debian/control - nbxmpp/__init__.py - nbxmpp/addresses.py - nbxmpp/client.py - nbxmpp/const.py - nbxmpp/examples/client.py - + nbxmpp/http.py - nbxmpp/modules/muclumbus.py - nbxmpp/structs.py - nbxmpp/task.py - nbxmpp/websocket.py - + test/unit/test_http.py - + test/unit/test_muclumbus.py Changes: ===================================== .gitlab-ci.yml ===================================== @@ -9,7 +9,7 @@ test-pylint: stage: test script: - python3 .ci/pylint-test.py - - coverage run --source=nbxmpp -m unittest discover -v + - NBXMPP_EXTERNAL_UNIT_TESTS=1 coverage run --source=nbxmpp -m unittest discover -v - coverage report -mi - coverage xml -i - pip install . ===================================== README.md ===================================== @@ -7,7 +7,7 @@ - python >= 3.9.0 - PyGObject - GLib >= 2.60 -- libsoup +- libsoup3 - precis-i18n - packaging - idna ===================================== debian/control ===================================== @@ -6,7 +6,7 @@ Build-Depends: debhelper-compat (= 13), dh-python, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-all, python3-gi, python3-idna, @@ -24,7 +24,7 @@ Architecture: all Depends: ${misc:Depends}, ${python3:Depends}, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-gi, python3-idna, python3-packaging, ===================================== nbxmpp/__init__.py ===================================== @@ -2,6 +2,6 @@ import gi from .protocol import * -gi.require_version('Soup', '2.4') +gi.require_version('Soup', '3.0') __version__: str = '3.2.5' ===================================== nbxmpp/addresses.py ===================================== @@ -18,9 +18,8 @@ import logging from collections import namedtuple -from gi.repository import Soup - -import nbxmpp +from nbxmpp.http import HTTPRequest +from nbxmpp.http import HTTPSession from nbxmpp.util import Observable from nbxmpp.const import ConnectionType from nbxmpp.const import ConnectionProtocol @@ -63,6 +62,7 @@ class ServerAddresses(Observable): Observable.__init__(self, log) self._domain = domain + self._http_session = None self._custom_host = None self._proxy = None self._is_resolved = False @@ -136,32 +136,31 @@ class ServerAddresses(Observable): self._resolve_alternatives() def _resolve_alternatives(self) -> None: - session = Soup.Session() - session.props.timeout = 5 - session.props.user_agent = f'nbxmpp/{nbxmpp.__version__}' - message = Soup.Message.new( - 'GET', f'https://{self._domain}/.well-known/host-meta') - session.queue_message(message, self._on_alternatives_result) - - def _on_alternatives_result(self, - _session: Soup.Session, - message: Soup.Message) -> None: - - status = message.props.status_code - if status != Soup.Status.OK: - error = message.props.reason_phrase - log.info('Failed to retrieve host-meta file: %s %s', status, error) + if self._http_session is None: self._on_request_resolved() return - response_body = message.props.response_body - if response_body is None or not response_body.data: + request = self._http_session.create_request() + request.send('GET', f'https://{self._domain}/.well-known/host-meta', + timeout=5, + callback=self._on_alternatives_result) + + def _on_alternatives_result(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + log.info('Failed to retrieve host-meta file: %s', + request.get_error_string()) + self._on_request_resolved() + return + + response_body = request.get_data() + if not response_body: log.info('No response body data found') self._on_request_resolved() return try: - uri = parse_websocket_uri(response_body.data) + uri = parse_websocket_uri(response_body.decode()) except Exception as error: log.info('Error parsing websocket uri: %s', error) self._on_request_resolved() @@ -215,6 +214,9 @@ class ServerAddresses(Observable): def set_proxy(self, proxy): self._proxy = proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + def _on_request_resolved(self): self._is_resolved = True self.notify('resolved') ===================================== nbxmpp/client.py ===================================== @@ -22,6 +22,7 @@ from typing import Optional from gi.repository import GLib from nbxmpp.namespaces import Namespace +from nbxmpp.http import HTTPSession from nbxmpp.protocol import Features from nbxmpp.protocol import StanzaMalformed from nbxmpp.protocol import SessionRequest @@ -93,6 +94,7 @@ class Client(Observable): self._client_cert = None self._client_cert_pass = None self._proxy = None + self._http_session = HTTPSession() self._allowed_con_types = None self._allowed_protocols = None @@ -329,12 +331,18 @@ class Client(Observable): def set_proxy(self, proxy): self._proxy = proxy - self._dispatcher.get_module('Muclumbus').set_proxy(proxy) @property def proxy(self): return self._proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + + @property + def http_session(self) -> HTTPSession: + return self._http_session + def get_bound_jid(self): return self._jid @@ -402,6 +410,7 @@ class Client(Observable): self.state = StreamState.RESOLVE self._addresses = ServerAddresses(self._domain) + self._addresses.set_http_session(self._http_session) self._addresses.set_custom_host(self._custom_host) self._addresses.set_proxy(self._proxy) self._addresses.subscribe('resolved', self._on_addresses_resolved) ===================================== nbxmpp/const.py ===================================== @@ -453,6 +453,16 @@ class Mode(IntEnum): return self == Mode.ANONYMOUS_TEST +class HTTPRequestError(IntEnum): + UNKNOWN = 0 + MISSING_CONTENT_LENGTH = 1 + INCOMPLETE = 2 + STATUS_NOT_OK = 3 + CANCELLED = 4 + CONTENT_OVERFLOW = 5 + TIMEOUT = 6 + + MOODS = [ 'afraid', 'amazed', ===================================== nbxmpp/examples/client.py ===================================== @@ -6,6 +6,7 @@ import json from pathlib import Path import gi + gi.require_version('Gtk', '3.0') gi.require_version('GLib', '2.0') from gi.repository import Gtk @@ -14,6 +15,7 @@ from gi.repository import GLib import nbxmpp from nbxmpp.protocol import JID from nbxmpp.client import Client +from nbxmpp.http import HTTPSession from nbxmpp.structs import ProxyData from nbxmpp.structs import StanzaHandler from nbxmpp.const import ConnectionType @@ -83,6 +85,7 @@ class TestClient(Gtk.Window): self._client.set_domain(self.address.domain) self._client.set_username(self.address.localpart) self._client.set_resource('test') + self._client.set_http_session(HTTPSession()) proxy_ip = self._builder.proxy_ip.get_text() if proxy_ip: ===================================== nbxmpp/http.py ===================================== @@ -0,0 +1,474 @@ +# This file is part of nbxmpp. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see <http://www.gnu.org/licenses/>. + +from __future__ import annotations + +from pathlib import Path + +from typing import Any +from typing import cast +from typing import Literal +from typing import Callable +from typing import Optional + +import logging + +from gi.repository import Soup +from gi.repository import Gio +from gi.repository import GLib +from gi.repository import GObject + +import nbxmpp +from .const import HTTPRequestError + + +log = logging.getLogger('nbxmpp.http') + + +HTTP_METHODS_T = Literal[ + 'CONNECT', + 'DELETE', + 'GET', + 'HEAD', + 'OPTIONS', + 'PATCH', + 'POST', + 'PUT', + 'TRACE', +] +CHUNK_SIZE = 32768 +DEFAULT_USER_AGENT = f'nbxmpp/{nbxmpp.__version__}' +SIGNAL_ACTIONS = GObject.SignalFlags.RUN_LAST | GObject.SignalFlags.ACTION + + +class HTTPLogAdapter(logging.LoggerAdapter): + def process(self, msg: str, kwargs: Any) -> tuple[str, Any]: + return f'{self.extra["request"]}: {msg}', kwargs + + +class HTTPSession: + def __init__(self, + user_agent: str = DEFAULT_USER_AGENT, + sniffer: bool = False + ) -> None: + + self._session = Soup.Session() + self._session.set_user_agent(user_agent) + + if sniffer: + self._session.add_feature_by_type(Soup.ContentSniffer) + + def get_soup_session(self) -> Soup.Session: + return self._session + + def set_proxy_resolver(self, + resolver: Optional[Gio.SimpleProxyResolver] + ) -> None: + + self._session.set_proxy_resolver(resolver) + + def create_request(self) -> HTTPRequest: + return HTTPRequest(self) + + +class HTTPRequest(GObject.GObject): + + __gtype_name__ = "HTTPRequest" + + __gsignals__ = { + 'starting-response-body': (SIGNAL_ACTIONS, None, ()), + 'response-progress': (SIGNAL_ACTIONS, None, (float,)), + 'request-progress': (SIGNAL_ACTIONS, None, (float,)), + 'finished': (SIGNAL_ACTIONS, None, ()), + 'destroy': (SIGNAL_ACTIONS, None, ()), + } + + def __init__(self, session: HTTPSession) -> None: + GObject.GObject.__init__(self) + + self._log = HTTPLogAdapter(log, extra={'request': self}) + + self._session = session + + self._received_size = 0 + self._sent_size = 0 + + self._cancellable = Gio.Cancellable() + self._input_stream = cast(Gio.InputStream, None) + self._output_stream: Optional[Gio.OutputStream] = None + self._is_finished = False + self._error: Optional[HTTPRequestError] = None + self._is_complete = False + self._timeout_reached = False + self._timeout_id = None + + self._response_body_file: Optional[Gio.File] = None + self._response_body_data = b'' + + self._request_body_file: Optional[Gio.File] = None + self._request_body_data: Optional[bytes] = None + + self._request_content_type = '' + self._request_content_length = 0 + + self._response_content_type = '' + self._response_content_length = 0 + + self._emit_request_progress = False + self._emit_response_progress = False + + self._message = Soup.Message() + self._user_data = None + + self._log.info('Created') + + def is_finished(self) -> bool: + return self._is_finished + + def is_complete(self) -> bool: + return self._is_complete + + def get_error(self) -> Optional[HTTPRequestError]: + return self._error + + def get_error_string(self) -> str: + if self._error == HTTPRequestError.STATUS_NOT_OK: + return self._message.get_reason_phrase() + return f'{self._error}' + + def get_response_headers(self) -> Soup.MessageHeaders: + return self._message.get_response_headers() + + def get_request_headers(self) -> Soup.MessageHeaders: + return self._message.get_request_headers() + + def get_data(self) -> bytes: + if not self._is_finished: + raise ValueError('Process not finished, data not available') + return self._response_body_data + + def get_uri(self) -> Optional[GLib.Uri]: + return self._message.get_uri() + + def get_status(self) -> Soup.Status: + return self._message.get_status() + + def get_soup_message(self) -> Soup.Message: + return self._message + + def set_user_data(self, user_data: Any) -> None: + self._user_data = user_data + + def get_user_data(self) -> Any: + return self._user_data + + def cancel(self) -> None: + if self._is_finished: + raise ValueError('Session already finished') + + self._cancellable.cancel() + + def set_request_body_from_path(self, content_type: str, path: Path) -> None: + if not path.exists(): + raise ValueError('%s does not exist' % path) + + if not path.is_file(): + raise ValueError('%s is not a file' % path) + + self._request_body_file = Gio.File.new_for_path(str(path)) + self._request_content_type = content_type + self._request_content_length = path.stat().st_size + + def set_request_body(self, content_type: str, data: bytes) -> None: + self._request_body_data = data + self._request_content_type = content_type + self._request_content_length = len(data) + + def set_response_body_from_path(self, path: Path) -> None: + self._response_body_file = Gio.File.new_for_path(str(path)) + + def connect(self, + signal_name: str, + callback: Any, + *args: Any + ) -> None: + + if signal_name == 'response-progress': + self._emit_response_progress = True + + if signal_name == 'request-progress': + self._emit_request_progress = True + + if signal_name in GObject.signal_list_names(HTTPRequest): + GObject.GObject.connect(self, signal_name, callback, *args) + return + + user_data = (callback, args) + + self._message.connect(signal_name, + self._on_connect_callback, + user_data) + + def _on_connect_callback(self, _message: Soup.Message, *args: Any) -> None: + signal_args = args[:-1] + callback, user_data = args[-1] + callback(self, *signal_args, *user_data) + + def send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None, + callback: Optional[Callable[[HTTPRequest], Any]] = None + ) -> None: + + if callback: + self.connect('finished', callback) + self._send(method, uri_string, timeout) + + def _send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None + ) -> None: + + if self._is_finished: + raise ValueError('Session already finished') + + self._message.set_method(method) + + if self._response_body_file is not None: + self._output_stream = self._response_body_file.replace( + None, + False, + Gio.FileCreateFlags.REPLACE_DESTINATION, + self._cancellable) + + uri = GLib.Uri.parse(uri_string, GLib.UriFlags.HAS_PASSWORD) + self._message.set_uri(uri) + + if self._request_body_data is not None: + self._message.set_request_body_from_bytes( + self._request_content_type, + GLib.Bytes.new(self._request_body_data) + ) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + if self._request_body_file is not None: + request_input_stream = self._request_body_file.read( + self._cancellable) + self._message.set_request_body(self._request_content_type, + request_input_stream, + self._request_content_length) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + self._message.connect('finished', self._on_finished) + + soup_session = self._session.get_soup_session() + soup_session.send_async(self._message, + GLib.PRIORITY_DEFAULT, + self._cancellable, + self._on_response) + + if timeout is not None: + self._timeout_id = GLib.timeout_add_seconds( + timeout, self._on_timeout) + + self._log.info('Request sent') + + def _on_request_body_progress(self, + _message: Soup.Message, + chunk: int) -> None: + + self._sent_size += chunk + self.emit('request-progress', + self._sent_size / self._request_content_length) + + def _on_timeout(self) -> None: + self._timeout_reached = True + self.cancel() + + def _on_response(self, + session: Soup.Session, + result: Gio.AsyncResult + ) -> None: + + self._log.info('Request response received') + try: + self._input_stream = session.send_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._set_failed(HTTPRequestError.CANCELLED) + else: + self._set_failed(HTTPRequestError.UNKNOWN) + return + + if self._message.get_status() not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + headers = self.get_response_headers() + self._response_content_length = headers.get_content_length() + self._response_content_type, _params = headers.get_content_type() + if self._response_content_length == 0: + self._set_failed(HTTPRequestError.MISSING_CONTENT_LENGTH) + return + + self._log.info('Start downloading response body') + self.emit('starting-response-body') + + self._read_async() + + def _read_async(self) -> None: + self._input_stream.read_bytes_async(CHUNK_SIZE, + GLib.PRIORITY_LOW, + self._cancellable, + self._on_bytes_read_result) + + def _on_bytes_read_result(self, + input_stream: Gio.InputStream, + result: Gio.AsyncResult) -> None: + try: + data = input_stream.read_bytes_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + + return + + bytes_ = data.get_data() + if not bytes_: + self._finish_read() + return + + length = len(bytes_) + self._received_size += length + if self._received_size > self._response_content_length: + self._finish_read(HTTPRequestError.CONTENT_OVERFLOW) + return + + if self._output_stream is None: + self._response_body_data += bytes_ + + else: + try: + self._output_stream.write_all(bytes_, self._cancellable) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + return + + self._read_async() + + if (self._emit_response_progress and + self._message.get_method() == 'GET'): + self.emit('response-progress', + self._received_size / self._response_content_length) + + + def _finish_read(self, error: Optional[HTTPRequestError] = None) -> None: + self._log.info('Finished reading') + if error is None: + self._close_all_streams() + return + + self._set_failed(error) + + def _on_finished(self, _message: Soup.Message) -> None: + self._set_finished() + + def _set_finished(self) -> None: + status = self._message.get_status() + if status == Soup.Status.NONE: + # Message has not been sent, can happen when we cancel the message + # before it is sent. The finished signal triggers before the + # cancelled exception. + return + + if self._is_finished: + return + + headers = self._message.get_response_headers() + content_length = headers.get_content_length() + if self._received_size != content_length: + self._set_failed(HTTPRequestError.INCOMPLETE) + return + + if status not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + self._set_complete() + + def _set_failed(self, error: HTTPRequestError) -> None: + self._is_finished = True + if self._timeout_reached: + self._timeout_id = None + self._error = HTTPRequestError.TIMEOUT + else: + self._error = error + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _set_complete(self) -> None: + self._is_finished = True + self._is_complete = True + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _close_all_streams(self) -> None: + if self._input_stream is not None: + if not self._input_stream.is_closed(): + self._input_stream.close(None) + + if self._output_stream is not None: + if not self._output_stream.is_closed(): + self._output_stream.close(None) + + def _cleanup(self) -> None: + self._log.info('Run cleanup') + + self._response_body_data = b'' + self._message.run_dispose() + + del self._cancellable + del self._session + del self._input_stream + del self._output_stream + del self._user_data + + if self._timeout_id is not None: + GLib.source_remove(self._timeout_id) + + self.emit('destroy') + self.run_dispose() + + def __repr__(self) -> str: + return f'Request({id(self)})' ===================================== nbxmpp/modules/muclumbus.py ===================================== @@ -17,8 +17,6 @@ import json -from gi.repository import Soup - from nbxmpp.namespaces import Namespace from nbxmpp.protocol import Node from nbxmpp.protocol import Iq @@ -44,15 +42,6 @@ class Muclumbus(BaseModule): self._client = client self.handlers = [] - self._proxy_resolver = None - self._soup_session = Soup.Session() - - def set_proxy(self, proxy): - if proxy is None: - return - self._proxy_resolver = proxy.get_resolver() - self._soup_session.props.proxy_resolver = self._proxy_resolver - @iq_request_task def request_parameters(self, jid): task = yield @@ -141,24 +130,25 @@ class Muclumbus(BaseModule): if after is not None: search['after'] = after - message = Soup.Message.new('POST', uri) - message.set_request('application/json', - Soup.MemoryUse.COPY, - json.dumps(search).encode()) + body = json.dumps(search).encode() - response_message = yield message + session = self._client.http_session + request = session.create_request() + request.set_request_body('application/json', body) + request.send('POST', uri) - soup_body = response_message.get_property('response-body') + request = yield request - if response_message.status_code != 200: - self._log.warning(soup_body.data) + if not request.is_complete(): + self._log.warning(request.get_error_string()) yield MuclumbusResult(first=None, last=None, max=None, end=True, items=[]) - response = json.loads(soup_body.data) + response_body = request.get_data() + response = json.loads(response_body) result = response['result'] items = result.get('items') ===================================== nbxmpp/structs.py ===================================== @@ -30,8 +30,8 @@ from dataclasses import dataclass from dataclasses import field from datetime import datetime -from gi.repository import Soup from gi.repository import Gio +from gi.repository import GLib from nbxmpp.simplexml import Node from nbxmpp.namespaces import Namespace @@ -708,8 +708,9 @@ class ProxyData(NamedTuple): def get_uri(self) -> str: if self.username is not None: - user_pass = Soup.uri_encode('%s:%s' % (self.username, - self.password)) + username = GLib.uri_escape_string(self.username, None, False) + password = GLib.uri_escape_string(self.password, None, False) + user_pass = f'{username}:{password}' return '%s://%s@%s' % (self.type, user_pass, self.host) ===================================== nbxmpp/task.py ===================================== @@ -27,14 +27,14 @@ import logging from enum import IntEnum from functools import wraps -from gi.repository import Soup from gi.repository import GLib -from nbxmpp.modules.base import BaseModule from nbxmpp.simplexml import Node from nbxmpp.errors import is_error from nbxmpp.errors import CancelledError from nbxmpp.errors import TimeoutStanzaError +from nbxmpp.http import HTTPRequest +from nbxmpp.modules.base import BaseModule from nbxmpp.modules.util import make_func_arguments_string @@ -109,8 +109,7 @@ def http_request_task(func): @wraps(func) def func_wrapper(self, *args, callback=None, user_data=None, **kwargs): task = HTTPRequestTask(func(self, *args, **kwargs), - self._log, - self._soup_session) + self._log) return _setup_task(task, self._client, callback, user_data) return func_wrapper @@ -251,7 +250,7 @@ class Task: def _invoke_callbacks(self): for callback in self._done_callbacks: - if isinstance(callback, weakref.WeakMethod): + if isinstance(callback, (weakref.WeakMethod, weakref.ref)): callback = callback() if callback is None: return @@ -377,19 +376,21 @@ class HTTPRequestTask(Task): ''' - _process_types = (Soup.Message,) + _process_types = (HTTPRequest,) - def __init__(self, gen, logger, session): + def __init__(self, gen, logger): super().__init__(gen, logger) - self._session = session - def _run_async(self, message): + def _run_async(self, request: HTTPRequest): # pylint: disable=arguments-renamed - self._session.queue_message(message, self._async_finished, None) + request.connect('finished', self._async_finished) + + def _async_finished(self, request: HTTPRequest): + + if self._state == TaskState.CANCELLED: + return - def _async_finished(self, _session, message, _user_data): - if self._state != TaskState.CANCELLED: - self._next_step(message) + self._next_step(request) def _finalize(self): self._session = None ===================================== nbxmpp/websocket.py ===================================== @@ -22,7 +22,6 @@ from gi.repository import GLib from gi.repository import Gio from nbxmpp.const import TCPState -from nbxmpp.const import ConnectionType from nbxmpp.util import get_websocket_close_string from nbxmpp.util import convert_tls_error_flags from nbxmpp.connection import Connection @@ -35,11 +34,10 @@ class WebsocketConnection(Connection): Connection.__init__(self, *args, **kwargs) self._session = Soup.Session() - self._session.props.ssl_strict = False if self._log.getEffectiveLevel() == logging.INFO: self._session.add_feature( - Soup.Logger.new(Soup.LoggerLogLevel.BODY, -1)) + Soup.Logger.new(Soup.LoggerLogLevel.BODY)) self._websocket = None self._cancellable = Gio.Cancellable() @@ -53,11 +51,12 @@ class WebsocketConnection(Connection): self.state = TCPState.CONNECTING message = Soup.Message.new('GET', self._address.uri) - message.connect('starting', self._check_certificate) + message.connect('accept-certificate', self._check_certificate) message.set_flags(Soup.MessageFlags.NO_REDIRECT) self._session.websocket_connect_async(message, None, ['xmpp'], + GLib.PRIORITY_DEFAULT, self._cancellable, self._on_connect, None) @@ -91,19 +90,20 @@ class WebsocketConnection(Connection): # Soup.Session does this automatically raise NotImplementedError - def _check_certificate(self, message): - https_used, certificate, errors = message.get_https_status() - if not https_used and self._address.type == ConnectionType.PLAIN: - return + def _check_certificate(self, + _message, + certificate, + errors): self._peer_certificate = certificate self._peer_certificate_errors = convert_tls_error_flags(errors) if self._accept_certificate(): - return + return True self.notify('bad-certificate') self._cancellable.cancel() + return False def _on_websocket_message(self, _websocket, _type, message): data = message.get_data().decode() ===================================== test/unit/test_http.py ===================================== @@ -0,0 +1,247 @@ +from time import time +import tempfile +import unittest +from unittest.mock import call +from unittest.mock import Mock +from unittest.mock import patch +import os +from pathlib import Path + +from gi.repository import GLib +from gi.repository import Soup + +from nbxmpp.const import HTTPRequestError +from nbxmpp.http import HTTPSession + + +SMALL_FILE_URL = 'https://gajim.org/downloads/ci/unittest_small_file' # 200 KB +BIG_FILE_URL = 'https://gajim.org/downloads/ci/unittest_big_file' # 7 MB +LARGE_FILE_URL = 'https://gajim.org/downloads/ci/unittest_large_file' # 80 MB +NO_FILE_URL = 'https://gajim.org/downloads/ci/no-file' + + +# import logging +# consoleloghandler = logging.StreamHandler() +# log = logging.getLogger('nbxmpp.http') +# log.setLevel(logging.DEBUG) +# log.addHandler(consoleloghandler) + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class HTTP(unittest.TestCase): + + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def test_download_to_file(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + + temp_dir = tempfile.gettempdir() + request.set_response_body_from_path(Path(temp_dir) / 'file') + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_in_memory_download(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_before_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + GLib.timeout_add(10, request.cancel) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.progress.assert_not_called() + callback_mock.starting.assert_not_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_after_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + def _on_progress(req, progress): + self.assertIsInstance(progress, float) + req.cancel() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('response-progress', _on_progress) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', LARGE_FILE_URL, timeout=10) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_failed_404(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('finished', callback_mock.finished) + request.connect('response-progress', callback_mock.progress) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', NO_FILE_URL, timeout=5) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.STATUS_NOT_OK) + self.assertEqual(request.get_status(), Soup.Status.NOT_FOUND) + + callback_mock.progress.assert_not_called() + callback_mock.finished.assert_called() + + def test_cancel_with_timeout(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', LARGE_FILE_URL, timeout=1) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertFalse(request.is_complete()) + self.assertEqual(request.get_error(), HTTPRequestError.TIMEOUT) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_signal_propagation(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting', callback_mock.starting, 1, 2, 3) + request.connect('got-body', callback_mock.got_body) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertTrue(request.is_complete()) + + callback_mock.starting.assert_called_with(request, 1, 2, 3) + callback_mock.got_body.assert_called_with(request) + + def test_parallel_download(self): + + mainloop = GLib.MainLoop() + + request1 = HTTPSession().create_request() + request1.send('GET', SMALL_FILE_URL, timeout=5) + + request2 = HTTPSession().create_request() + request2.send('GET', SMALL_FILE_URL, timeout=5) + + request3 = HTTPSession().create_request() + request3.send('GET', SMALL_FILE_URL, timeout=5) + + request4 = HTTPSession().create_request() + request4.send('GET', SMALL_FILE_URL, timeout=5) + + GLib.timeout_add_seconds(5, mainloop.quit) + + mainloop.run() + + self.assertTrue(request1.is_finished()) + self.assertTrue(request1.is_complete()) + + self.assertTrue(request2.is_finished()) + self.assertTrue(request2.is_complete()) + + self.assertTrue(request3.is_finished()) + self.assertTrue(request3.is_complete()) + + self.assertTrue(request4.is_finished()) + self.assertTrue(request4.is_complete()) + + +if __name__ == '__main__': + unittest.main() ===================================== test/unit/test_muclumbus.py ===================================== @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Any + +from gi.repository import GLib + +import os +import unittest +from unittest.mock import Mock +from nbxmpp.client import Client +from nbxmpp.http import HTTPSession +from nbxmpp.structs import MuclumbusResult +from nbxmpp.modules.muclumbus import Muclumbus + +from nbxmpp.sasl import SCRAM_SHA_1 +from nbxmpp.util import b64encode + +# Test vector from https://wiki.xmpp.org/web/SASL_and_SCRAM-SHA-1 + +API_URL = 'https://search.jabber.network/api/1.0/search' + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class TestMuclumbus(unittest.TestCase): + def setUp(self): + self._client = Client() + self._client.set_http_session(HTTPSession()) + self._module = Muclumbus(self._client) + + def test_http_request(self): + + mainloop = GLib.MainLoop() + + def _result(task: Any) -> None: + result = task.finish() + assert isinstance(result, MuclumbusResult) + assert len(result.items) > 0 + mainloop.quit() + + self._module.set_http_search(API_URL, 'gajim', callback=_result) + + mainloop.run() + +if __name__ == '__main__': + unittest.main() View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/0da8888db96f23a7329802af… -- View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/0da8888db96f23a7329802af… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/python-nbxmpp][soup3] change: Port to libsoup3
by Philipp Hörist (＠lovetox) 28 Dec '22

28 Dec '22

Philipp Hörist pushed to branch soup3 at gajim / python-nbxmpp Commits: ddb9c25e by Philipp Hörist at 2022-12-28T09:00:11+01:00 change: Port to libsoup3 - Port code to libsoup3 - Wrap Soup.Session and Soup.Message for easier usage - - - - - 15 changed files: - .gitlab-ci.yml - README.md - debian/control - nbxmpp/__init__.py - nbxmpp/addresses.py - nbxmpp/client.py - nbxmpp/const.py - nbxmpp/examples/client.py - + nbxmpp/http.py - nbxmpp/modules/muclumbus.py - nbxmpp/structs.py - nbxmpp/task.py - nbxmpp/websocket.py - + test/unit/test_http.py - + test/unit/test_muclumbus.py Changes: ===================================== .gitlab-ci.yml ===================================== @@ -9,7 +9,7 @@ test-pylint: stage: test script: - python3 .ci/pylint-test.py - - coverage run --source=nbxmpp -m unittest discover -v + - NBXMPP_EXTERNAL_UNIT_TESTS=1 coverage run --source=nbxmpp -m unittest discover -v - coverage report -mi - coverage xml -i - pip install . ===================================== README.md ===================================== @@ -7,7 +7,7 @@ - python >= 3.9.0 - PyGObject - GLib >= 2.60 -- libsoup +- libsoup3 - precis-i18n - packaging - idna ===================================== debian/control ===================================== @@ -6,7 +6,7 @@ Build-Depends: debhelper-compat (= 13), dh-python, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-all, python3-gi, python3-idna, @@ -24,7 +24,7 @@ Architecture: all Depends: ${misc:Depends}, ${python3:Depends}, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-gi, python3-idna, python3-packaging, ===================================== nbxmpp/__init__.py ===================================== @@ -2,6 +2,6 @@ import gi from .protocol import * -gi.require_version('Soup', '2.4') +gi.require_version('Soup', '3.0') __version__: str = '3.2.5' ===================================== nbxmpp/addresses.py ===================================== @@ -18,9 +18,8 @@ import logging from collections import namedtuple -from gi.repository import Soup - -import nbxmpp +from nbxmpp.http import HTTPRequest +from nbxmpp.http import HTTPSession from nbxmpp.util import Observable from nbxmpp.const import ConnectionType from nbxmpp.const import ConnectionProtocol @@ -63,6 +62,7 @@ class ServerAddresses(Observable): Observable.__init__(self, log) self._domain = domain + self._http_session = None self._custom_host = None self._proxy = None self._is_resolved = False @@ -136,32 +136,31 @@ class ServerAddresses(Observable): self._resolve_alternatives() def _resolve_alternatives(self) -> None: - session = Soup.Session() - session.props.timeout = 5 - session.props.user_agent = f'nbxmpp/{nbxmpp.__version__}' - message = Soup.Message.new( - 'GET', f'https://{self._domain}/.well-known/host-meta') - session.queue_message(message, self._on_alternatives_result) - - def _on_alternatives_result(self, - _session: Soup.Session, - message: Soup.Message) -> None: - - status = message.props.status_code - if status != Soup.Status.OK: - error = message.props.reason_phrase - log.info('Failed to retrieve host-meta file: %s %s', status, error) + if self._http_session is None: self._on_request_resolved() return - response_body = message.props.response_body - if response_body is None or not response_body.data: + request = self._http_session.create_request() + request.send('GET', f'https://{self._domain}/.well-known/host-meta', + timeout=5, + callback=self._on_alternatives_result) + + def _on_alternatives_result(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + log.info('Failed to retrieve host-meta file: %s', + request.get_error_string()) + self._on_request_resolved() + return + + response_body = request.get_data() + if not response_body: log.info('No response body data found') self._on_request_resolved() return try: - uri = parse_websocket_uri(response_body.data) + uri = parse_websocket_uri(response_body.decode()) except Exception as error: log.info('Error parsing websocket uri: %s', error) self._on_request_resolved() @@ -215,6 +214,9 @@ class ServerAddresses(Observable): def set_proxy(self, proxy): self._proxy = proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + def _on_request_resolved(self): self._is_resolved = True self.notify('resolved') ===================================== nbxmpp/client.py ===================================== @@ -22,6 +22,7 @@ from typing import Optional from gi.repository import GLib from nbxmpp.namespaces import Namespace +from nbxmpp.http import HTTPSession from nbxmpp.protocol import Features from nbxmpp.protocol import StanzaMalformed from nbxmpp.protocol import SessionRequest @@ -93,6 +94,7 @@ class Client(Observable): self._client_cert = None self._client_cert_pass = None self._proxy = None + self._http_session = HTTPSession() self._allowed_con_types = None self._allowed_protocols = None @@ -329,12 +331,18 @@ class Client(Observable): def set_proxy(self, proxy): self._proxy = proxy - self._dispatcher.get_module('Muclumbus').set_proxy(proxy) @property def proxy(self): return self._proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + + @property + def http_session(self) -> HTTPSession: + return self._http_session + def get_bound_jid(self): return self._jid @@ -402,6 +410,7 @@ class Client(Observable): self.state = StreamState.RESOLVE self._addresses = ServerAddresses(self._domain) + self._addresses.set_http_session(self._http_session) self._addresses.set_custom_host(self._custom_host) self._addresses.set_proxy(self._proxy) self._addresses.subscribe('resolved', self._on_addresses_resolved) ===================================== nbxmpp/const.py ===================================== @@ -453,6 +453,16 @@ class Mode(IntEnum): return self == Mode.ANONYMOUS_TEST +class HTTPRequestError(IntEnum): + UNKNOWN = 0 + MISSING_CONTENT_LENGTH = 1 + INCOMPLETE = 2 + STATUS_NOT_OK = 3 + CANCELLED = 4 + CONTENT_OVERFLOW = 5 + TIMEOUT = 6 + + MOODS = [ 'afraid', 'amazed', ===================================== nbxmpp/examples/client.py ===================================== @@ -6,6 +6,7 @@ import json from pathlib import Path import gi + gi.require_version('Gtk', '3.0') gi.require_version('GLib', '2.0') from gi.repository import Gtk @@ -14,6 +15,7 @@ from gi.repository import GLib import nbxmpp from nbxmpp.protocol import JID from nbxmpp.client import Client +from nbxmpp.http import HTTPSession from nbxmpp.structs import ProxyData from nbxmpp.structs import StanzaHandler from nbxmpp.const import ConnectionType @@ -83,6 +85,7 @@ class TestClient(Gtk.Window): self._client.set_domain(self.address.domain) self._client.set_username(self.address.localpart) self._client.set_resource('test') + self._client.set_http_session(HTTPSession()) proxy_ip = self._builder.proxy_ip.get_text() if proxy_ip: ===================================== nbxmpp/http.py ===================================== @@ -0,0 +1,473 @@ +# This file is part of nbxmpp. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see <http://www.gnu.org/licenses/>. + +from __future__ import annotations + +from pathlib import Path + +from typing import Any +from typing import cast +from typing import Literal +from typing import Callable +from typing import Optional + +import logging + +from gi.repository import Soup +from gi.repository import Gio +from gi.repository import GLib +from gi.repository import GObject + +import nbxmpp +from .const import HTTPRequestError + + +log = logging.getLogger('nbxmpp.http') + + +HTTP_METHODS_T = Literal[ + 'CONNECT', + 'DELETE', + 'GET', + 'HEAD', + 'OPTIONS', + 'PATCH', + 'POST', + 'PUT', + 'TRACE', +] +CHUNK_SIZE = 32768 +DEFAULT_USER_AGENT = f'nbxmpp/{nbxmpp.__version__}' +SIGNAL_ACTIONS = GObject.SignalFlags.RUN_LAST | GObject.SignalFlags.ACTION + + +class HTTPLogAdapter(logging.LoggerAdapter): + def process(self, msg: str, kwargs: Any) -> tuple[str, Any]: + return f'{self.extra["request"]}: {msg}', kwargs + + +class HTTPSession: + def __init__(self, + user_agent: str = DEFAULT_USER_AGENT, + sniffer: bool = False + ) -> None: + + self._session = Soup.Session() + self._session.set_user_agent(user_agent) + + if sniffer: + self._session.add_feature_by_type(Soup.ContentSniffer) + + def get_soup_session(self) -> Soup.Session: + return self._session + + def set_proxy_resolver(self, + resolver: Optional[Gio.SimpleProxyResolver] + ) -> None: + + self._session.set_proxy_resolver(resolver) + + def create_request(self) -> HTTPRequest: + return HTTPRequest(self) + + +class HTTPRequest(GObject.GObject): + + __gtype_name__ = "HTTPRequest" + + __gsignals__ = { + 'starting-response-body': (SIGNAL_ACTIONS, None, ()), + 'response-progress': (SIGNAL_ACTIONS, None, (float,)), + 'request-progress': (SIGNAL_ACTIONS, None, (float,)), + 'finished': (SIGNAL_ACTIONS, None, ()), + 'destroy': (SIGNAL_ACTIONS, None, ()), + } + + def __init__(self, session: HTTPSession) -> None: + GObject.GObject.__init__(self) + + self._log = HTTPLogAdapter(log, extra={'request': self}) + + self._session = session + + self._received_size = 0 + self._sent_size = 0 + + self._cancellable = Gio.Cancellable() + self._input_stream = cast(Gio.InputStream, None) + self._output_stream: Optional[Gio.OutputStream] = None + self._is_finished = False + self._error: Optional[HTTPRequestError] = None + self._is_complete = False + self._timeout_reached = False + self._timeout_id = None + + self._response_body_file: Optional[Gio.File] = None + self._response_body_data = b'' + + self._request_body_file: Optional[Gio.File] = None + self._request_body_data: Optional[bytes] = None + + self._request_content_type = '' + self._request_content_length = 0 + + self._response_content_type = '' + self._response_content_length = 0 + + self._emit_request_progress = False + self._emit_response_progress = False + + self._message = Soup.Message() + self._user_data = None + + self._log.info('Created') + + def is_finished(self) -> bool: + return self._is_finished + + def is_complete(self) -> bool: + return self._is_complete + + def get_error(self) -> Optional[HTTPRequestError]: + return self._error + + def get_error_string(self) -> str: + if self._error == HTTPRequestError.STATUS_NOT_OK: + return self._message.get_reason_phrase() + return f'{self._error}' + + def get_response_headers(self) -> Soup.MessageHeaders: + return self._message.get_response_headers() + + def get_request_headers(self) -> Soup.MessageHeaders: + return self._message.get_request_headers() + + def get_data(self) -> bytes: + if not self._is_finished: + raise ValueError('Process not finished, data not available') + return self._response_body_data + + def get_uri(self) -> Optional[GLib.Uri]: + return self._message.get_uri() + + def get_status(self) -> Soup.Status: + return self._message.get_status() + + def get_soup_message(self) -> Soup.Message: + return self._message + + def set_user_data(self, user_data: Any) -> None: + self._user_data = user_data + + def get_user_data(self) -> Any: + return self._user_data + + def cancel(self) -> None: + if self._is_finished: + raise ValueError('Session already finished') + + self._cancellable.cancel() + + def set_request_body_from_path(self, content_type: str, path: Path) -> None: + if not path.exists(): + raise ValueError('%s does not exist' % path) + + if not path.is_file(): + raise ValueError('%s is not a file' % path) + + self._request_body_file = Gio.File.new_for_path(str(path)) + self._request_content_type = content_type + self._request_content_length = path.stat().st_size + + def set_request_body(self, content_type: str, data: bytes) -> None: + self._request_body_data = data + self._request_content_type = content_type + self._request_content_length = len(data) + + def set_response_body_from_path(self, path: Path) -> None: + self._response_body_file = Gio.File.new_for_path(str(path)) + + def connect(self, + signal_name: str, + callback: Any, + *args: Any + ) -> None: + + if signal_name == 'response-progress': + self._emit_response_progress = True + + if signal_name == 'request-progress': + self._emit_request_progress = True + + if signal_name in GObject.signal_list_names(HTTPRequest): + GObject.GObject.connect(self, signal_name, callback, *args) + return + + user_data = (callback, args) + + self._message.connect(signal_name, + self._on_connect_callback, + user_data) + + def _on_connect_callback(self, _message: Soup.Message, *args: Any) -> None: + signal_args = args[:-1] + callback, user_data = args[-1] + callback(self, *signal_args, *user_data) + + def send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None, + callback: Optional[Callable[[HTTPRequest], Any]] = None + ) -> None: + + if callback: + self.connect('finished', callback) + self._send(method, uri_string, timeout) + + def _send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None + ) -> None: + + if self._is_finished: + raise ValueError('Session already finished') + + self._message.set_method(method) + + if self._response_body_file is not None: + self._output_stream = self._response_body_file.replace( + None, + False, + Gio.FileCreateFlags.REPLACE_DESTINATION, + self._cancellable) + + uri = GLib.Uri.parse(uri_string, GLib.UriFlags.HAS_PASSWORD) + self._message.set_uri(uri) + + if self._request_body_data is not None: + self._message.set_request_body_from_bytes( + self._request_content_type, + GLib.Bytes.new(self._request_body_data) + ) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + if self._request_body_file is not None: + request_input_stream = self._request_body_file.read( + self._cancellable) + self._message.set_request_body(self._request_content_type, + request_input_stream, + self._request_content_length) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + self._message.connect('finished', self._on_finished) + + soup_session = self._session.get_soup_session() + soup_session.send_async(self._message, + GLib.PRIORITY_DEFAULT, + self._cancellable, + self._on_response) + + if timeout is not None: + self._timeout_id = GLib.timeout_add_seconds( + timeout, self._on_timeout) + + self._log.info('Request sent') + + def _on_request_body_progress(self, + _message: Soup.Message, + chunk: int) -> None: + self._sent_size =+ chunk + self.emit('request-progress', + self._sent_size / self._request_content_length) + + def _on_timeout(self) -> None: + self._timeout_reached = True + self.cancel() + + def _on_response(self, + session: Soup.Session, + result: Gio.AsyncResult + ) -> None: + + self._log.info('Request response received') + try: + self._input_stream = session.send_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._set_failed(HTTPRequestError.CANCELLED) + else: + self._set_failed(HTTPRequestError.UNKNOWN) + return + + if self._message.get_status() not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + headers = self.get_response_headers() + self._response_content_length = headers.get_content_length() + self._response_content_type, _params = headers.get_content_type() + if self._response_content_length == 0: + self._set_failed(HTTPRequestError.MISSING_CONTENT_LENGTH) + return + + self._log.info('Start downloading response body') + self.emit('starting-response-body') + + self._read_async() + + def _read_async(self) -> None: + self._input_stream.read_bytes_async(CHUNK_SIZE, + GLib.PRIORITY_LOW, + self._cancellable, + self._on_bytes_read_result) + + def _on_bytes_read_result(self, + input_stream: Gio.InputStream, + result: Gio.AsyncResult) -> None: + try: + data = input_stream.read_bytes_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + + return + + bytes_ = data.get_data() + if not bytes_: + self._finish_read() + return + + length = len(bytes_) + self._received_size += length + if self._received_size > self._response_content_length: + self._finish_read(HTTPRequestError.CONTENT_OVERFLOW) + return + + if self._output_stream is None: + self._response_body_data += bytes_ + + else: + try: + self._output_stream.write_all(bytes_, self._cancellable) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + return + + self._read_async() + + if (self._emit_response_progress and + self._message.get_method() == 'GET'): + self.emit('response-progress', + self._received_size / self._response_content_length) + + + def _finish_read(self, error: Optional[HTTPRequestError] = None) -> None: + self._log.info('Finished reading') + if error is None: + self._close_all_streams() + return + + self._set_failed(error) + + def _on_finished(self, _message: Soup.Message) -> None: + self._set_finished() + + def _set_finished(self) -> None: + status = self._message.get_status() + if status == Soup.Status.NONE: + # Message has not been sent, can happen when we cancel the message + # before it is sent. The finished signal triggers before the + # cancelled exception. + return + + if self._is_finished: + return + + headers = self._message.get_response_headers() + content_length = headers.get_content_length() + if self._received_size != content_length: + self._set_failed(HTTPRequestError.INCOMPLETE) + return + + if status not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + self._set_complete() + + def _set_failed(self, error: HTTPRequestError) -> None: + self._is_finished = True + if self._timeout_reached: + self._timeout_id = None + self._error = HTTPRequestError.TIMEOUT + else: + self._error = error + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _set_complete(self) -> None: + self._is_finished = True + self._is_complete = True + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _close_all_streams(self) -> None: + if self._input_stream is not None: + if not self._input_stream.is_closed(): + self._input_stream.close(None) + + if self._output_stream is not None: + if not self._output_stream.is_closed(): + self._output_stream.close(None) + + def _cleanup(self) -> None: + self._log.info('Run cleanup') + + self._response_body_data = b'' + self._message.run_dispose() + + del self._cancellable + del self._session + del self._input_stream + del self._output_stream + del self._user_data + + if self._timeout_id is not None: + GLib.source_remove(self._timeout_id) + + self.emit('destroy') + self.run_dispose() + + def __repr__(self) -> str: + return f'Request({id(self)})' ===================================== nbxmpp/modules/muclumbus.py ===================================== @@ -17,8 +17,6 @@ import json -from gi.repository import Soup - from nbxmpp.namespaces import Namespace from nbxmpp.protocol import Node from nbxmpp.protocol import Iq @@ -44,15 +42,6 @@ class Muclumbus(BaseModule): self._client = client self.handlers = [] - self._proxy_resolver = None - self._soup_session = Soup.Session() - - def set_proxy(self, proxy): - if proxy is None: - return - self._proxy_resolver = proxy.get_resolver() - self._soup_session.props.proxy_resolver = self._proxy_resolver - @iq_request_task def request_parameters(self, jid): task = yield @@ -141,24 +130,25 @@ class Muclumbus(BaseModule): if after is not None: search['after'] = after - message = Soup.Message.new('POST', uri) - message.set_request('application/json', - Soup.MemoryUse.COPY, - json.dumps(search).encode()) + body = json.dumps(search).encode() - response_message = yield message + session = self._client.http_session + request = session.create_request() + request.set_request_body('application/json', body) + request.send('POST', uri) - soup_body = response_message.get_property('response-body') + request = yield request - if response_message.status_code != 200: - self._log.warning(soup_body.data) + if not request.is_complete(): + self._log.warning(request.get_error_string()) yield MuclumbusResult(first=None, last=None, max=None, end=True, items=[]) - response = json.loads(soup_body.data) + response_body = request.get_data() + response = json.loads(response_body) result = response['result'] items = result.get('items') ===================================== nbxmpp/structs.py ===================================== @@ -30,8 +30,8 @@ from dataclasses import dataclass from dataclasses import field from datetime import datetime -from gi.repository import Soup from gi.repository import Gio +from gi.repository import GLib from nbxmpp.simplexml import Node from nbxmpp.namespaces import Namespace @@ -708,8 +708,9 @@ class ProxyData(NamedTuple): def get_uri(self) -> str: if self.username is not None: - user_pass = Soup.uri_encode('%s:%s' % (self.username, - self.password)) + username = GLib.uri_escape_string(self.username, None, False) + password = GLib.uri_escape_string(self.password, None, False) + user_pass = f'{username}:{password}' return '%s://%s@%s' % (self.type, user_pass, self.host) ===================================== nbxmpp/task.py ===================================== @@ -27,14 +27,14 @@ import logging from enum import IntEnum from functools import wraps -from gi.repository import Soup from gi.repository import GLib -from nbxmpp.modules.base import BaseModule from nbxmpp.simplexml import Node from nbxmpp.errors import is_error from nbxmpp.errors import CancelledError from nbxmpp.errors import TimeoutStanzaError +from nbxmpp.http import HTTPRequest +from nbxmpp.modules.base import BaseModule from nbxmpp.modules.util import make_func_arguments_string @@ -109,8 +109,7 @@ def http_request_task(func): @wraps(func) def func_wrapper(self, *args, callback=None, user_data=None, **kwargs): task = HTTPRequestTask(func(self, *args, **kwargs), - self._log, - self._soup_session) + self._log) return _setup_task(task, self._client, callback, user_data) return func_wrapper @@ -251,7 +250,7 @@ class Task: def _invoke_callbacks(self): for callback in self._done_callbacks: - if isinstance(callback, weakref.WeakMethod): + if isinstance(callback, (weakref.WeakMethod, weakref.ref)): callback = callback() if callback is None: return @@ -377,19 +376,21 @@ class HTTPRequestTask(Task): ''' - _process_types = (Soup.Message,) + _process_types = (HTTPRequest,) - def __init__(self, gen, logger, session): + def __init__(self, gen, logger): super().__init__(gen, logger) - self._session = session - def _run_async(self, message): + def _run_async(self, request: HTTPRequest): # pylint: disable=arguments-renamed - self._session.queue_message(message, self._async_finished, None) + request.connect('finished', self._async_finished) + + def _async_finished(self, request: HTTPRequest): + + if self._state == TaskState.CANCELLED: + return - def _async_finished(self, _session, message, _user_data): - if self._state != TaskState.CANCELLED: - self._next_step(message) + self._next_step(request) def _finalize(self): self._session = None ===================================== nbxmpp/websocket.py ===================================== @@ -22,7 +22,6 @@ from gi.repository import GLib from gi.repository import Gio from nbxmpp.const import TCPState -from nbxmpp.const import ConnectionType from nbxmpp.util import get_websocket_close_string from nbxmpp.util import convert_tls_error_flags from nbxmpp.connection import Connection @@ -35,11 +34,10 @@ class WebsocketConnection(Connection): Connection.__init__(self, *args, **kwargs) self._session = Soup.Session() - self._session.props.ssl_strict = False if self._log.getEffectiveLevel() == logging.INFO: self._session.add_feature( - Soup.Logger.new(Soup.LoggerLogLevel.BODY, -1)) + Soup.Logger.new(Soup.LoggerLogLevel.BODY)) self._websocket = None self._cancellable = Gio.Cancellable() @@ -53,11 +51,12 @@ class WebsocketConnection(Connection): self.state = TCPState.CONNECTING message = Soup.Message.new('GET', self._address.uri) - message.connect('starting', self._check_certificate) + message.connect('accept-certificate', self._check_certificate) message.set_flags(Soup.MessageFlags.NO_REDIRECT) self._session.websocket_connect_async(message, None, ['xmpp'], + GLib.PRIORITY_DEFAULT, self._cancellable, self._on_connect, None) @@ -91,19 +90,20 @@ class WebsocketConnection(Connection): # Soup.Session does this automatically raise NotImplementedError - def _check_certificate(self, message): - https_used, certificate, errors = message.get_https_status() - if not https_used and self._address.type == ConnectionType.PLAIN: - return + def _check_certificate(self, + _message, + certificate, + errors): self._peer_certificate = certificate self._peer_certificate_errors = convert_tls_error_flags(errors) if self._accept_certificate(): - return + return True self.notify('bad-certificate') self._cancellable.cancel() + return False def _on_websocket_message(self, _websocket, _type, message): data = message.get_data().decode() ===================================== test/unit/test_http.py ===================================== @@ -0,0 +1,247 @@ +from time import time +import tempfile +import unittest +from unittest.mock import call +from unittest.mock import Mock +from unittest.mock import patch +import os +from pathlib import Path + +from gi.repository import GLib +from gi.repository import Soup + +from nbxmpp.const import HTTPRequestError +from nbxmpp.http import HTTPSession + + +SMALL_FILE_URL = 'https://gajim.org/downloads/ci/unittest_small_file' # 200 KB +BIG_FILE_URL = 'https://gajim.org/downloads/ci/unittest_big_file' # 7 MB +LARGE_FILE_URL = 'https://gajim.org/downloads/ci/unittest_large_file' # 80 MB +NO_FILE_URL = 'https://gajim.org/downloads/ci/no-file' + + +# import logging +# consoleloghandler = logging.StreamHandler() +# log = logging.getLogger('nbxmpp.http') +# log.setLevel(logging.DEBUG) +# log.addHandler(consoleloghandler) + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class HTTP(unittest.TestCase): + + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def test_download_to_file(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + + temp_dir = tempfile.gettempdir() + request.set_response_body_from_path(Path(temp_dir) / 'file') + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_in_memory_download(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_before_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + GLib.timeout_add(10, request.cancel) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.progress.assert_not_called() + callback_mock.starting.assert_not_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_after_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + def _on_progress(req, progress): + self.assertIsInstance(progress, float) + req.cancel() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('response-progress', _on_progress) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', LARGE_FILE_URL, timeout=10) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_failed_404(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('finished', callback_mock.finished) + request.connect('response-progress', callback_mock.progress) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', NO_FILE_URL, timeout=5) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.STATUS_NOT_OK) + self.assertEqual(request.get_status(), Soup.Status.NOT_FOUND) + + callback_mock.progress.assert_not_called() + callback_mock.finished.assert_called() + + def test_cancel_with_timeout(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', LARGE_FILE_URL, timeout=1) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertFalse(request.is_complete()) + self.assertEqual(request.get_error(), HTTPRequestError.TIMEOUT) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_signal_propagation(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting', callback_mock.starting, 1, 2, 3) + request.connect('got-body', callback_mock.got_body) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertTrue(request.is_complete()) + + callback_mock.starting.assert_called_with(request, 1, 2, 3) + callback_mock.got_body.assert_called_with(request) + + def test_parallel_download(self): + + mainloop = GLib.MainLoop() + + request1 = HTTPSession().create_request() + request1.send('GET', SMALL_FILE_URL, timeout=5) + + request2 = HTTPSession().create_request() + request2.send('GET', SMALL_FILE_URL, timeout=5) + + request3 = HTTPSession().create_request() + request3.send('GET', SMALL_FILE_URL, timeout=5) + + request4 = HTTPSession().create_request() + request4.send('GET', SMALL_FILE_URL, timeout=5) + + GLib.timeout_add_seconds(5, mainloop.quit) + + mainloop.run() + + self.assertTrue(request1.is_finished()) + self.assertTrue(request1.is_complete()) + + self.assertTrue(request2.is_finished()) + self.assertTrue(request2.is_complete()) + + self.assertTrue(request3.is_finished()) + self.assertTrue(request3.is_complete()) + + self.assertTrue(request4.is_finished()) + self.assertTrue(request4.is_complete()) + + +if __name__ == '__main__': + unittest.main() ===================================== test/unit/test_muclumbus.py ===================================== @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Any + +from gi.repository import GLib + +import os +import unittest +from unittest.mock import Mock +from nbxmpp.client import Client +from nbxmpp.http import HTTPSession +from nbxmpp.structs import MuclumbusResult +from nbxmpp.modules.muclumbus import Muclumbus + +from nbxmpp.sasl import SCRAM_SHA_1 +from nbxmpp.util import b64encode + +# Test vector from https://wiki.xmpp.org/web/SASL_and_SCRAM-SHA-1 + +API_URL = 'https://search.jabber.network/api/1.0/search' + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class TestMuclumbus(unittest.TestCase): + def setUp(self): + self._client = Client() + self._client.set_http_session(HTTPSession()) + self._module = Muclumbus(self._client) + + def test_http_request(self): + + mainloop = GLib.MainLoop() + + def _result(task: Any) -> None: + result = task.finish() + assert isinstance(result, MuclumbusResult) + assert len(result.items) > 0 + mainloop.quit() + + self._module.set_http_search(API_URL, 'gajim', callback=_result) + + mainloop.run() + +if __name__ == '__main__': + unittest.main() View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/ddb9c25e0e23bc5644e5289c… -- View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/ddb9c25e0e23bc5644e5289c… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/python-nbxmpp][soup3] change: Port to libsoup3
by Philipp Hörist (＠lovetox) 28 Dec '22

28 Dec '22

Philipp Hörist pushed to branch soup3 at gajim / python-nbxmpp Commits: 7b583ac4 by Philipp Hörist at 2022-12-28T08:45:53+01:00 change: Port to libsoup3 - Port code to libsoup3 - Wrap Soup.Session and Soup.Message for easier usage - - - - - 15 changed files: - .gitlab-ci.yml - README.md - debian/control - nbxmpp/__init__.py - nbxmpp/addresses.py - nbxmpp/client.py - nbxmpp/const.py - nbxmpp/examples/client.py - + nbxmpp/http.py - nbxmpp/modules/muclumbus.py - nbxmpp/structs.py - nbxmpp/task.py - nbxmpp/websocket.py - + test/unit/test_http.py - + test/unit/test_muclumbus.py Changes: ===================================== .gitlab-ci.yml ===================================== @@ -9,7 +9,7 @@ test-pylint: stage: test script: - python3 .ci/pylint-test.py - - coverage run --source=nbxmpp -m unittest discover -v + - NBXMPP_EXTERNAL_UNIT_TESTS=1 coverage run --source=nbxmpp -m unittest discover -v - coverage report -mi - coverage xml -i - pip install . ===================================== README.md ===================================== @@ -7,7 +7,7 @@ - python >= 3.9.0 - PyGObject - GLib >= 2.60 -- libsoup +- libsoup3 - precis-i18n - packaging - idna ===================================== debian/control ===================================== @@ -6,7 +6,7 @@ Build-Depends: debhelper-compat (= 13), dh-python, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-all, python3-gi, python3-idna, @@ -24,7 +24,7 @@ Architecture: all Depends: ${misc:Depends}, ${python3:Depends}, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-gi, python3-idna, python3-packaging, ===================================== nbxmpp/__init__.py ===================================== @@ -2,6 +2,6 @@ import gi from .protocol import * -gi.require_version('Soup', '2.4') +gi.require_version('Soup', '3.0') __version__: str = '3.2.5' ===================================== nbxmpp/addresses.py ===================================== @@ -18,9 +18,8 @@ import logging from collections import namedtuple -from gi.repository import Soup - -import nbxmpp +from nbxmpp.http import HTTPRequest +from nbxmpp.http import HTTPSession from nbxmpp.util import Observable from nbxmpp.const import ConnectionType from nbxmpp.const import ConnectionProtocol @@ -63,6 +62,7 @@ class ServerAddresses(Observable): Observable.__init__(self, log) self._domain = domain + self._http_session = None self._custom_host = None self._proxy = None self._is_resolved = False @@ -136,32 +136,31 @@ class ServerAddresses(Observable): self._resolve_alternatives() def _resolve_alternatives(self) -> None: - session = Soup.Session() - session.props.timeout = 5 - session.props.user_agent = f'nbxmpp/{nbxmpp.__version__}' - message = Soup.Message.new( - 'GET', f'https://{self._domain}/.well-known/host-meta') - session.queue_message(message, self._on_alternatives_result) - - def _on_alternatives_result(self, - _session: Soup.Session, - message: Soup.Message) -> None: - - status = message.props.status_code - if status != Soup.Status.OK: - error = message.props.reason_phrase - log.info('Failed to retrieve host-meta file: %s %s', status, error) + if self._http_session is None: self._on_request_resolved() return - response_body = message.props.response_body - if response_body is None or not response_body.data: + request = self._http_session.create_request() + request.send('GET', f'https://{self._domain}/.well-known/host-meta', + timeout=5, + callback=self._on_alternatives_result) + + def _on_alternatives_result(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + log.info('Failed to retrieve host-meta file: %s', + request.get_error_string()) + self._on_request_resolved() + return + + response_body = request.get_data() + if not response_body: log.info('No response body data found') self._on_request_resolved() return try: - uri = parse_websocket_uri(response_body.data) + uri = parse_websocket_uri(response_body.decode()) except Exception as error: log.info('Error parsing websocket uri: %s', error) self._on_request_resolved() @@ -215,6 +214,9 @@ class ServerAddresses(Observable): def set_proxy(self, proxy): self._proxy = proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + def _on_request_resolved(self): self._is_resolved = True self.notify('resolved') ===================================== nbxmpp/client.py ===================================== @@ -22,6 +22,7 @@ from typing import Optional from gi.repository import GLib from nbxmpp.namespaces import Namespace +from nbxmpp.http import HTTPSession from nbxmpp.protocol import Features from nbxmpp.protocol import StanzaMalformed from nbxmpp.protocol import SessionRequest @@ -93,6 +94,7 @@ class Client(Observable): self._client_cert = None self._client_cert_pass = None self._proxy = None + self._http_session = HTTPSession() self._allowed_con_types = None self._allowed_protocols = None @@ -329,12 +331,18 @@ class Client(Observable): def set_proxy(self, proxy): self._proxy = proxy - self._dispatcher.get_module('Muclumbus').set_proxy(proxy) @property def proxy(self): return self._proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + + @property + def http_session(self) -> HTTPSession: + return self._http_session + def get_bound_jid(self): return self._jid @@ -402,6 +410,7 @@ class Client(Observable): self.state = StreamState.RESOLVE self._addresses = ServerAddresses(self._domain) + self._addresses.set_http_session(self._http_session) self._addresses.set_custom_host(self._custom_host) self._addresses.set_proxy(self._proxy) self._addresses.subscribe('resolved', self._on_addresses_resolved) ===================================== nbxmpp/const.py ===================================== @@ -453,6 +453,16 @@ class Mode(IntEnum): return self == Mode.ANONYMOUS_TEST +class HTTPRequestError(IntEnum): + UNKNOWN = 0 + MISSING_CONTENT_LENGTH = 1 + INCOMPLETE = 2 + STATUS_NOT_OK = 3 + CANCELLED = 4 + CONTENT_OVERFLOW = 5 + TIMEOUT = 6 + + MOODS = [ 'afraid', 'amazed', ===================================== nbxmpp/examples/client.py ===================================== @@ -6,6 +6,7 @@ import json from pathlib import Path import gi + gi.require_version('Gtk', '3.0') gi.require_version('GLib', '2.0') from gi.repository import Gtk @@ -14,6 +15,7 @@ from gi.repository import GLib import nbxmpp from nbxmpp.protocol import JID from nbxmpp.client import Client +from nbxmpp.http import HTTPSession from nbxmpp.structs import ProxyData from nbxmpp.structs import StanzaHandler from nbxmpp.const import ConnectionType @@ -83,6 +85,7 @@ class TestClient(Gtk.Window): self._client.set_domain(self.address.domain) self._client.set_username(self.address.localpart) self._client.set_resource('test') + self._client.set_http_session(HTTPSession()) proxy_ip = self._builder.proxy_ip.get_text() if proxy_ip: ===================================== nbxmpp/http.py ===================================== @@ -0,0 +1,473 @@ +# This file is part of nbxmpp. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see <http://www.gnu.org/licenses/>. + +from __future__ import annotations + +from pathlib import Path + +from typing import Any +from typing import cast +from typing import Literal +from typing import Callable +from typing import Optional + +import logging + +from gi.repository import Soup +from gi.repository import Gio +from gi.repository import GLib +from gi.repository import GObject + +import nbxmpp +from .const import HTTPRequestError + + +log = logging.getLogger('nbxmpp.http') + + +HTTP_METHODS_T = Literal[ + 'CONNECT', + 'DELETE', + 'GET', + 'HEAD', + 'OPTIONS', + 'PATCH', + 'POST', + 'PUT', + 'TRACE', +] +CHUNK_SIZE = 32768 +DEFAULT_USER_AGENT = f'nbxmpp/{nbxmpp.__version__}' +SIGNAL_ACTIONS = GObject.SignalFlags.RUN_LAST | GObject.SignalFlags.ACTION + + +class HTTPLogAdapter(logging.LoggerAdapter): + def process(self, msg: str, kwargs: Any) -> tuple[str, Any]: + return f'{self.extra["request"]}: {msg}', kwargs + + +class HTTPSession: + def __init__(self, + user_agent: str = DEFAULT_USER_AGENT, + sniffer: bool = False + ) -> None: + + self._session = Soup.Session() + self._session.set_user_agent(user_agent) + + if sniffer: + self._session.add_feature_by_type(Soup.ContentSniffer) + + def get_soup_session(self) -> Soup.Session: + return self._session + + def set_proxy_resolver(self, + resolver: Optional[Gio.SimpleProxyResolver] + ) -> None: + + self._session.set_proxy_resolver(resolver) + + def create_request(self) -> HTTPRequest: + return HTTPRequest(self) + + +class HTTPRequest(GObject.GObject): + + __gtype_name__ = "HTTPRequest" + + __gsignals__ = { + 'starting-response-body': (SIGNAL_ACTIONS, None, ()), + 'response-progress': (SIGNAL_ACTIONS, None, (float,)), + 'request-progress': (SIGNAL_ACTIONS, None, (float,)), + 'finished': (SIGNAL_ACTIONS, None, ()), + 'destroy': (SIGNAL_ACTIONS, None, ()), + } + + def __init__(self, session: HTTPSession) -> None: + GObject.GObject.__init__(self) + + self._log = HTTPLogAdapter(log, extra={'request': self}) + + self._session = session + + self._received_size = 0 + self._sent_size = 0 + + self._cancellable = Gio.Cancellable() + self._input_stream = cast(Gio.InputStream, None) + self._output_stream: Optional[Gio.OutputStream] = None + self._is_finished = False + self._error: Optional[HTTPRequestError] = None + self._is_complete = False + self._timeout_reached = False + self._timeout_id = None + + self._response_body_file: Optional[Gio.File] = None + self._response_body_data = b'' + + self._request_body_file: Optional[Gio.File] = None + self._request_body_data: Optional[bytes] = None + + self._request_content_type = '' + self._request_content_length = 0 + + self._response_content_type = '' + self._response_content_length = 0 + + self._emit_request_progress = False + self._emit_response_progress = False + + self._message = Soup.Message() + self._user_data = None + + self._log.info('Created') + + def is_finished(self) -> bool: + return self._is_finished + + def is_complete(self) -> bool: + return self._is_complete + + def get_error(self) -> Optional[HTTPRequestError]: + return self._error + + def get_error_string(self) -> str: + if self._error == HTTPRequestError.STATUS_NOT_OK: + return self._message.get_reason_phrase() + return f'{self._error}' + + def get_response_headers(self) -> Soup.MessageHeaders: + return self._message.get_response_headers() + + def get_request_headers(self) -> Soup.MessageHeaders: + return self._message.get_request_headers() + + def get_data(self) -> bytes: + if not self._is_finished: + raise ValueError('Process not finished, data not available') + return self._response_body_data + + def get_uri(self) -> Optional[GLib.Uri]: + return self._message.get_uri() + + def get_status(self) -> Soup.Status: + return self._message.get_status() + + def get_soup_message(self) -> Soup.Message: + return self._message + + def set_user_data(self, user_data: Any) -> None: + self._user_data = user_data + + def get_user_data(self) -> Any: + return self._user_data + + def cancel(self) -> None: + if self._is_finished: + raise ValueError('Session already finished') + + self._cancellable.cancel() + + def set_request_body_from_path(self, content_type: str, path: Path) -> None: + if not path.exists(): + raise ValueError('%s does not exist' % path) + + if not path.is_file(): + raise ValueError('%s is not a file' % path) + + self._request_body_file = Gio.File.new_for_path(str(path)) + self._request_content_type = content_type + self._request_content_length = path.stat().st_size + + def set_request_body(self, content_type: str, data: bytes) -> None: + self._request_body_data = data + self._request_content_type = content_type + self._request_content_length = len(data) + + def set_response_body_from_path(self, path: Path) -> None: + self._response_body_file = Gio.File.new_for_path(str(path)) + + def connect(self, + signal_name: str, + callback: Any, + *args: Any + ) -> None: + + if signal_name == 'response-progress': + self._emit_response_progress = True + + if signal_name == 'request-progress': + self._emit_request_progress = True + + if signal_name in GObject.signal_list_names(HTTPRequest): + GObject.GObject.connect(self, signal_name, callback, *args) + return + + user_data = (callback, args) + + self._message.connect(signal_name, + self._on_connect_callback, + user_data) + + def _on_connect_callback(self, _message: Soup.Message, *args: Any) -> None: + signal_args = args[:-1] + callback, user_data = args[-1] + callback(self, *signal_args, *user_data) + + def send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None, + callback: Optional[Callable[[HTTPRequest], Any]] = None + ) -> None: + + if callback: + self.connect('finished', callback) + self._send(method, uri_string, timeout) + + def _send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None + ) -> None: + + if self._is_finished: + raise ValueError('Session already finished') + + self._message.set_method(method) + + if self._response_body_file is not None: + self._output_stream = self._response_body_file.replace( + None, + False, + Gio.FileCreateFlags.REPLACE_DESTINATION, + self._cancellable) + + uri = GLib.Uri.parse(uri_string, GLib.UriFlags.HAS_PASSWORD) + self._message.set_uri(uri) + + if self._request_body_data is not None: + self._message.set_request_body_from_bytes( + self._request_content_type, + GLib.Bytes.new(self._request_body_data) + ) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + if self._request_body_file is not None: + request_input_stream = self._request_body_file.read( + self._cancellable) + self._message.set_request_body(self._request_content_type, + request_input_stream, + self._request_content_length) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + self._message.connect('finished', self._on_finished) + + soup_session = self._session.get_soup_session() + soup_session.send_async(self._message, + GLib.PRIORITY_DEFAULT, + self._cancellable, + self._on_response) + + if timeout is not None: + self._timeout_id = GLib.timeout_add_seconds( + timeout, self._on_timeout) + + self._log.info('Request sent') + + def _on_request_body_progress(self, + _message: Soup.Message, + chunk: int) -> None: + self._sent_size =+ chunk + self.emit('request-progress', + self._sent_size / self._request_content_length) + + def _on_timeout(self) -> None: + self._timeout_reached = True + self.cancel() + + def _on_response(self, + session: Soup.Session, + result: Gio.AsyncResult + ) -> None: + + self._log.info('Request response received') + try: + self._input_stream = session.send_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._set_failed(HTTPRequestError.CANCELLED) + else: + self._set_failed(HTTPRequestError.UNKNOWN) + return + + if self._message.get_status() not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + headers = self.get_response_headers() + self._response_content_length = headers.get_content_length() + self._response_content_type, _params = headers.get_content_type() + if self._response_content_length == 0: + self._set_failed(HTTPRequestError.MISSING_CONTENT_LENGTH) + return + + self._log.info('Start downloading response body') + self.emit('starting-response-body') + + self._read_async() + + def _read_async(self) -> None: + self._input_stream.read_bytes_async(CHUNK_SIZE, + GLib.PRIORITY_LOW, + self._cancellable, + self._on_bytes_read_result) + + def _on_bytes_read_result(self, + input_stream: Gio.InputStream, + result: Gio.AsyncResult) -> None: + try: + data = input_stream.read_bytes_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + + return + + bytes_ = data.get_data() + if not bytes_: + self._finish_read() + return + + length = len(bytes_) + self._received_size += length + if self._received_size > self._response_content_length: + self._finish_read(HTTPRequestError.CONTENT_OVERFLOW) + return + + if self._output_stream is None: + self._response_body_data += bytes_ + + else: + try: + self._output_stream.write_all(bytes_, self._cancellable) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + return + + self._read_async() + + if (self._emit_response_progress and + self._message.get_method() == 'GET'): + self.emit('response-progress', + self._received_size / self._response_content_length) + + + def _finish_read(self, error: Optional[HTTPRequestError] = None) -> None: + self._log.info('Finished reading') + if error is None: + self._close_all_streams() + return + + self._set_failed(error) + + def _on_finished(self, _message: Soup.Message) -> None: + self._set_finished() + + def _set_finished(self) -> None: + status = self._message.get_status() + if status == Soup.Status.NONE: + # Message has not been sent, can happen when we cancel the message + # before it is sent. The finished signal triggers before the + # cancelled exception. + return + + if self._is_finished: + return + + headers = self._message.get_response_headers() + content_length = headers.get_content_length() + if self._received_size != content_length: + self._set_failed(HTTPRequestError.INCOMPLETE) + return + + if status not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + self._set_complete() + + def _set_failed(self, error: HTTPRequestError) -> None: + self._is_finished = True + if self._timeout_reached: + self._timeout_id = None + self._error = HTTPRequestError.TIMEOUT + else: + self._error = error + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _set_complete(self) -> None: + self._is_finished = True + self._is_complete = True + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _close_all_streams(self) -> None: + if self._input_stream is not None: + if not self._input_stream.is_closed(): + self._input_stream.close(None) + + if self._output_stream is not None: + if not self._output_stream.is_closed(): + self._output_stream.close(None) + + def _cleanup(self) -> None: + self._log.info('Run cleanup') + + self._response_body_data = b'' + self._message.run_dispose() + + del self._cancellable + del self._session + del self._input_stream + del self._output_stream + del self._user_data + + if self._timeout_id is not None: + GLib.source_remove(self._timeout_id) + + self.emit('destroy') + self.run_dispose() + + def __repr__(self) -> str: + return f'Request({id(self)})' ===================================== nbxmpp/modules/muclumbus.py ===================================== @@ -17,8 +17,6 @@ import json -from gi.repository import Soup - from nbxmpp.namespaces import Namespace from nbxmpp.protocol import Node from nbxmpp.protocol import Iq @@ -44,15 +42,6 @@ class Muclumbus(BaseModule): self._client = client self.handlers = [] - self._proxy_resolver = None - self._soup_session = Soup.Session() - - def set_proxy(self, proxy): - if proxy is None: - return - self._proxy_resolver = proxy.get_resolver() - self._soup_session.props.proxy_resolver = self._proxy_resolver - @iq_request_task def request_parameters(self, jid): task = yield @@ -141,24 +130,25 @@ class Muclumbus(BaseModule): if after is not None: search['after'] = after - message = Soup.Message.new('POST', uri) - message.set_request('application/json', - Soup.MemoryUse.COPY, - json.dumps(search).encode()) + body = json.dumps(search).encode() - response_message = yield message + session = self._client.http_session + request = session.create_request() + request.set_request_body('application/json', body) + request.send('POST', uri) - soup_body = response_message.get_property('response-body') + request = yield request - if response_message.status_code != 200: - self._log.warning(soup_body.data) + if not request.is_complete(): + self._log.warning(request.get_error_string()) yield MuclumbusResult(first=None, last=None, max=None, end=True, items=[]) - response = json.loads(soup_body.data) + response_body = request.get_data() + response = json.loads(response_body) result = response['result'] items = result.get('items') ===================================== nbxmpp/structs.py ===================================== @@ -30,8 +30,8 @@ from dataclasses import dataclass from dataclasses import field from datetime import datetime -from gi.repository import Soup from gi.repository import Gio +from gi.repository import GLib from nbxmpp.simplexml import Node from nbxmpp.namespaces import Namespace @@ -708,8 +708,9 @@ class ProxyData(NamedTuple): def get_uri(self) -> str: if self.username is not None: - user_pass = Soup.uri_encode('%s:%s' % (self.username, - self.password)) + username = GLib.uri_escape_string(self.username, None, False) + password = GLib.uri_escape_string(self.password, None, False) + user_pass = f'{username}:{password}' return '%s://%s@%s' % (self.type, user_pass, self.host) ===================================== nbxmpp/task.py ===================================== @@ -27,14 +27,14 @@ import logging from enum import IntEnum from functools import wraps -from gi.repository import Soup from gi.repository import GLib -from nbxmpp.modules.base import BaseModule from nbxmpp.simplexml import Node from nbxmpp.errors import is_error from nbxmpp.errors import CancelledError from nbxmpp.errors import TimeoutStanzaError +from nbxmpp.http import HTTPRequest +from nbxmpp.modules.base import BaseModule from nbxmpp.modules.util import make_func_arguments_string @@ -109,8 +109,7 @@ def http_request_task(func): @wraps(func) def func_wrapper(self, *args, callback=None, user_data=None, **kwargs): task = HTTPRequestTask(func(self, *args, **kwargs), - self._log, - self._soup_session) + self._log) return _setup_task(task, self._client, callback, user_data) return func_wrapper @@ -251,7 +250,7 @@ class Task: def _invoke_callbacks(self): for callback in self._done_callbacks: - if isinstance(callback, weakref.WeakMethod): + if isinstance(callback, (weakref.WeakMethod, weakref.ref)): callback = callback() if callback is None: return @@ -377,19 +376,21 @@ class HTTPRequestTask(Task): ''' - _process_types = (Soup.Message,) + _process_types = (HTTPRequest,) - def __init__(self, gen, logger, session): + def __init__(self, gen, logger): super().__init__(gen, logger) - self._session = session - def _run_async(self, message): + def _run_async(self, request: HTTPRequest): # pylint: disable=arguments-renamed - self._session.queue_message(message, self._async_finished, None) + request.connect('finished', self._async_finished) + + def _async_finished(self, request: HTTPRequest): + + if self._state == TaskState.CANCELLED: + return - def _async_finished(self, _session, message, _user_data): - if self._state != TaskState.CANCELLED: - self._next_step(message) + self._next_step(request) def _finalize(self): self._session = None ===================================== nbxmpp/websocket.py ===================================== @@ -22,7 +22,6 @@ from gi.repository import GLib from gi.repository import Gio from nbxmpp.const import TCPState -from nbxmpp.const import ConnectionType from nbxmpp.util import get_websocket_close_string from nbxmpp.util import convert_tls_error_flags from nbxmpp.connection import Connection @@ -35,11 +34,10 @@ class WebsocketConnection(Connection): Connection.__init__(self, *args, **kwargs) self._session = Soup.Session() - self._session.props.ssl_strict = False if self._log.getEffectiveLevel() == logging.INFO: self._session.add_feature( - Soup.Logger.new(Soup.LoggerLogLevel.BODY, -1)) + Soup.Logger.new(Soup.LoggerLogLevel.BODY)) self._websocket = None self._cancellable = Gio.Cancellable() @@ -53,11 +51,12 @@ class WebsocketConnection(Connection): self.state = TCPState.CONNECTING message = Soup.Message.new('GET', self._address.uri) - message.connect('starting', self._check_certificate) + message.connect('accept-certificate', self._check_certificate) message.set_flags(Soup.MessageFlags.NO_REDIRECT) self._session.websocket_connect_async(message, None, ['xmpp'], + GLib.PRIORITY_DEFAULT, self._cancellable, self._on_connect, None) @@ -91,19 +90,20 @@ class WebsocketConnection(Connection): # Soup.Session does this automatically raise NotImplementedError - def _check_certificate(self, message): - https_used, certificate, errors = message.get_https_status() - if not https_used and self._address.type == ConnectionType.PLAIN: - return + def _check_certificate(self, + _message, + certificate, + errors): self._peer_certificate = certificate self._peer_certificate_errors = convert_tls_error_flags(errors) if self._accept_certificate(): - return + return True self.notify('bad-certificate') self._cancellable.cancel() + return False def _on_websocket_message(self, _websocket, _type, message): data = message.get_data().decode() ===================================== test/unit/test_http.py ===================================== @@ -0,0 +1,246 @@ +from time import time +import tempfile +import unittest +from unittest.mock import call +from unittest.mock import Mock +from unittest.mock import patch +import os +from pathlib import Path + +from gi.repository import GLib +from gi.repository import Soup + +from nbxmpp.const import HTTPRequestError +from nbxmpp.http import HTTPSession + + +SMALL_FILE_URL = 'https://gajim.org/downloads/ci/unittest_small_file' +BIG_FILE_URL = 'https://gajim.org/downloads/ci/unittest_big_file' +NO_FILE_URL = 'https://gajim.org/downloads/ci/no-file' + + +# import logging +# consoleloghandler = logging.StreamHandler() +# log = logging.getLogger('nbxmpp.http') +# log.setLevel(logging.DEBUG) +# log.addHandler(consoleloghandler) + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class HTTP(unittest.TestCase): + + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def test_download_to_file(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + + temp_dir = tempfile.gettempdir() + request.set_response_body_from_path(Path(temp_dir) / 'file') + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_in_memory_download(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_before_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + GLib.timeout_add(10, request.cancel) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.progress.assert_not_called() + callback_mock.starting.assert_not_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_after_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + def _on_progress(req, progress): + self.assertIsInstance(progress, float) + req.cancel() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('response-progress', _on_progress) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', BIG_FILE_URL, timeout=10) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_failed_404(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('finished', callback_mock.finished) + request.connect('response-progress', callback_mock.progress) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', NO_FILE_URL, timeout=5) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.STATUS_NOT_OK) + self.assertEqual(request.get_status(), Soup.Status.NOT_FOUND) + + callback_mock.progress.assert_not_called() + callback_mock.finished.assert_called() + + def test_cancel_with_timeout(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', BIG_FILE_URL, timeout=1) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertFalse(request.is_complete()) + self.assertEqual(request.get_error(), HTTPRequestError.TIMEOUT) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_signal_propagation(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting', callback_mock.starting, 1, 2, 3) + request.connect('got-body', callback_mock.got_body) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertTrue(request.is_complete()) + + callback_mock.starting.assert_called_with(request, 1, 2, 3) + callback_mock.got_body.assert_called_with(request) + + def test_parallel_download(self): + + mainloop = GLib.MainLoop() + + request1 = HTTPSession().create_request() + request1.send('GET', SMALL_FILE_URL, timeout=5) + + request2 = HTTPSession().create_request() + request2.send('GET', SMALL_FILE_URL, timeout=5) + + request3 = HTTPSession().create_request() + request3.send('GET', SMALL_FILE_URL, timeout=5) + + request4 = HTTPSession().create_request() + request4.send('GET', SMALL_FILE_URL, timeout=5) + + GLib.timeout_add_seconds(5, mainloop.quit) + + mainloop.run() + + self.assertTrue(request1.is_finished()) + self.assertTrue(request1.is_complete()) + + self.assertTrue(request2.is_finished()) + self.assertTrue(request2.is_complete()) + + self.assertTrue(request3.is_finished()) + self.assertTrue(request3.is_complete()) + + self.assertTrue(request4.is_finished()) + self.assertTrue(request4.is_complete()) + + +if __name__ == '__main__': + unittest.main() ===================================== test/unit/test_muclumbus.py ===================================== @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Any + +from gi.repository import GLib + +import os +import unittest +from unittest.mock import Mock +from nbxmpp.client import Client +from nbxmpp.http import HTTPSession +from nbxmpp.structs import MuclumbusResult +from nbxmpp.modules.muclumbus import Muclumbus + +from nbxmpp.sasl import SCRAM_SHA_1 +from nbxmpp.util import b64encode + +# Test vector from https://wiki.xmpp.org/web/SASL_and_SCRAM-SHA-1 + +API_URL = 'https://search.jabber.network/api/1.0/search' + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class TestMuclumbus(unittest.TestCase): + def setUp(self): + self._client = Client() + self._client.set_http_session(HTTPSession()) + self._module = Muclumbus(self._client) + + def test_http_request(self): + + mainloop = GLib.MainLoop() + + def _result(task: Any) -> None: + result = task.finish() + assert isinstance(result, MuclumbusResult) + assert len(result.items) > 0 + mainloop.quit() + + self._module.set_http_search(API_URL, 'gajim', callback=_result) + + mainloop.run() + +if __name__ == '__main__': + unittest.main() View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/7b583ac4c37b718c5e9d8d93… -- View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/7b583ac4c37b718c5e9d8d93… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/python-nbxmpp][soup3] Deleted 1 commit: other: 4.0.0.dev1
by Philipp Hörist (＠lovetox) 27 Dec '22

27 Dec '22

Philipp Hörist pushed to branch soup3 at gajim / python-nbxmpp WARNING: The push did not contain any new commits, but force pushed to delete the commits and changes below. Deleted commits: 4ea91338 by Philipp Hörist at 2022-12-27T22:47:42+01:00 other: 4.0.0.dev1 - - - - - 2 changed files: - nbxmpp/__init__.py - setup.cfg Changes: ===================================== nbxmpp/__init__.py ===================================== @@ -2,6 +2,6 @@ import gi from .protocol import * -gi.require_version('Soup', '2.4') +gi.require_version('Soup', '3.0') -__version__: str = '3.2.5' +__version__: str = '4.0.0.dev1' ===================================== setup.cfg ===================================== @@ -1,6 +1,6 @@ [metadata] name = nbxmpp -version = 3.2.5 +version = 4.0.0.dev1 description = XMPP Library author = Yann Leboulanger, Philipp Hoerist author_email = gajim-devel(a)gajim.org View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/4ea913387059916b140ea419… -- View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/4ea913387059916b140ea419… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/python-nbxmpp][soup3] 2 commits: change: Port to libsoup3
by Philipp Hörist (＠lovetox) 27 Dec '22

27 Dec '22

Philipp Hörist pushed to branch soup3 at gajim / python-nbxmpp Commits: 564af7bb by Philipp Hörist at 2022-12-27T22:47:16+01:00 change: Port to libsoup3 - Port code to libsoup3 - Wrap Soup.Session and Soup.Message for easier usage - - - - - 4ea91338 by Philipp Hörist at 2022-12-27T22:47:42+01:00 other: 4.0.0.dev1 - - - - - 16 changed files: - .gitlab-ci.yml - README.md - debian/control - nbxmpp/__init__.py - nbxmpp/addresses.py - nbxmpp/client.py - nbxmpp/const.py - nbxmpp/examples/client.py - + nbxmpp/http.py - nbxmpp/modules/muclumbus.py - nbxmpp/structs.py - nbxmpp/task.py - nbxmpp/websocket.py - setup.cfg - + test/unit/test_http.py - + test/unit/test_muclumbus.py Changes: ===================================== .gitlab-ci.yml ===================================== @@ -9,7 +9,7 @@ test-pylint: stage: test script: - python3 .ci/pylint-test.py - - coverage run --source=nbxmpp -m unittest discover -v + - NBXMPP_EXTERNAL_UNIT_TESTS=1 coverage run --source=nbxmpp -m unittest discover -v - coverage report -mi - coverage xml -i - pip install . ===================================== README.md ===================================== @@ -7,7 +7,7 @@ - python >= 3.9.0 - PyGObject - GLib >= 2.60 -- libsoup +- libsoup3 - precis-i18n - packaging - idna ===================================== debian/control ===================================== @@ -6,7 +6,7 @@ Build-Depends: debhelper-compat (= 13), dh-python, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-all, python3-gi, python3-idna, @@ -24,7 +24,7 @@ Architecture: all Depends: ${misc:Depends}, ${python3:Depends}, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-gi, python3-idna, python3-packaging, ===================================== nbxmpp/__init__.py ===================================== @@ -2,6 +2,6 @@ import gi from .protocol import * -gi.require_version('Soup', '2.4') +gi.require_version('Soup', '3.0') -__version__: str = '3.2.5' +__version__: str = '4.0.0.dev1' ===================================== nbxmpp/addresses.py ===================================== @@ -18,9 +18,8 @@ import logging from collections import namedtuple -from gi.repository import Soup - -import nbxmpp +from nbxmpp.http import HTTPRequest +from nbxmpp.http import HTTPSession from nbxmpp.util import Observable from nbxmpp.const import ConnectionType from nbxmpp.const import ConnectionProtocol @@ -63,6 +62,7 @@ class ServerAddresses(Observable): Observable.__init__(self, log) self._domain = domain + self._http_session = None self._custom_host = None self._proxy = None self._is_resolved = False @@ -136,32 +136,31 @@ class ServerAddresses(Observable): self._resolve_alternatives() def _resolve_alternatives(self) -> None: - session = Soup.Session() - session.props.timeout = 5 - session.props.user_agent = f'nbxmpp/{nbxmpp.__version__}' - message = Soup.Message.new( - 'GET', f'https://{self._domain}/.well-known/host-meta') - session.queue_message(message, self._on_alternatives_result) - - def _on_alternatives_result(self, - _session: Soup.Session, - message: Soup.Message) -> None: - - status = message.props.status_code - if status != Soup.Status.OK: - error = message.props.reason_phrase - log.info('Failed to retrieve host-meta file: %s %s', status, error) + if self._http_session is None: self._on_request_resolved() return - response_body = message.props.response_body - if response_body is None or not response_body.data: + request = self._http_session.create_request() + request.send('GET', f'https://{self._domain}/.well-known/host-meta', + timeout=5, + callback=self._on_alternatives_result) + + def _on_alternatives_result(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + log.info('Failed to retrieve host-meta file: %s', + request.get_error_string()) + self._on_request_resolved() + return + + response_body = request.get_data() + if not response_body: log.info('No response body data found') self._on_request_resolved() return try: - uri = parse_websocket_uri(response_body.data) + uri = parse_websocket_uri(response_body.decode()) except Exception as error: log.info('Error parsing websocket uri: %s', error) self._on_request_resolved() @@ -215,6 +214,9 @@ class ServerAddresses(Observable): def set_proxy(self, proxy): self._proxy = proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + def _on_request_resolved(self): self._is_resolved = True self.notify('resolved') ===================================== nbxmpp/client.py ===================================== @@ -22,6 +22,7 @@ from typing import Optional from gi.repository import GLib from nbxmpp.namespaces import Namespace +from nbxmpp.http import HTTPSession from nbxmpp.protocol import Features from nbxmpp.protocol import StanzaMalformed from nbxmpp.protocol import SessionRequest @@ -93,6 +94,7 @@ class Client(Observable): self._client_cert = None self._client_cert_pass = None self._proxy = None + self._http_session = HTTPSession() self._allowed_con_types = None self._allowed_protocols = None @@ -329,12 +331,18 @@ class Client(Observable): def set_proxy(self, proxy): self._proxy = proxy - self._dispatcher.get_module('Muclumbus').set_proxy(proxy) @property def proxy(self): return self._proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + + @property + def http_session(self) -> HTTPSession: + return self._http_session + def get_bound_jid(self): return self._jid @@ -402,6 +410,7 @@ class Client(Observable): self.state = StreamState.RESOLVE self._addresses = ServerAddresses(self._domain) + self._addresses.set_http_session(self._http_session) self._addresses.set_custom_host(self._custom_host) self._addresses.set_proxy(self._proxy) self._addresses.subscribe('resolved', self._on_addresses_resolved) ===================================== nbxmpp/const.py ===================================== @@ -453,6 +453,16 @@ class Mode(IntEnum): return self == Mode.ANONYMOUS_TEST +class HTTPRequestError(IntEnum): + UNKNOWN = 0 + MISSING_CONTENT_LENGTH = 1 + INCOMPLETE = 2 + STATUS_NOT_OK = 3 + CANCELLED = 4 + CONTENT_OVERFLOW = 5 + TIMEOUT = 6 + + MOODS = [ 'afraid', 'amazed', ===================================== nbxmpp/examples/client.py ===================================== @@ -6,6 +6,7 @@ import json from pathlib import Path import gi + gi.require_version('Gtk', '3.0') gi.require_version('GLib', '2.0') from gi.repository import Gtk @@ -14,6 +15,7 @@ from gi.repository import GLib import nbxmpp from nbxmpp.protocol import JID from nbxmpp.client import Client +from nbxmpp.http import HTTPSession from nbxmpp.structs import ProxyData from nbxmpp.structs import StanzaHandler from nbxmpp.const import ConnectionType @@ -83,6 +85,7 @@ class TestClient(Gtk.Window): self._client.set_domain(self.address.domain) self._client.set_username(self.address.localpart) self._client.set_resource('test') + self._client.set_http_session(HTTPSession()) proxy_ip = self._builder.proxy_ip.get_text() if proxy_ip: ===================================== nbxmpp/http.py ===================================== @@ -0,0 +1,473 @@ +# This file is part of nbxmpp. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see <http://www.gnu.org/licenses/>. + +from __future__ import annotations + +from pathlib import Path + +from typing import Any +from typing import cast +from typing import Literal +from typing import Callable +from typing import Optional + +import logging + +from gi.repository import Soup +from gi.repository import Gio +from gi.repository import GLib +from gi.repository import GObject + +import nbxmpp +from .const import HTTPRequestError + + +log = logging.getLogger('nbxmpp.http') + + +HTTP_METHODS_T = Literal[ + 'CONNECT', + 'DELETE', + 'GET', + 'HEAD', + 'OPTIONS', + 'PATCH', + 'POST', + 'PUT', + 'TRACE', +] +CHUNK_SIZE = 32768 +DEFAULT_USER_AGENT = f'nbxmpp/{nbxmpp.__version__}' +SIGNAL_ACTIONS = GObject.SignalFlags.RUN_LAST | GObject.SignalFlags.ACTION + + +class HTTPLogAdapter(logging.LoggerAdapter): + def process(self, msg: str, kwargs: Any) -> tuple[str, Any]: + return f'{self.extra["request"]}: {msg}', kwargs + + +class HTTPSession: + def __init__(self, + user_agent: str = DEFAULT_USER_AGENT, + sniffer: bool = False + ) -> None: + + self._session = Soup.Session() + self._session.set_user_agent(user_agent) + + if sniffer: + self._session.add_feature_by_type(Soup.ContentSniffer) + + def get_soup_session(self) -> Soup.Session: + return self._session + + def set_proxy_resolver(self, + resolver: Optional[Gio.SimpleProxyResolver] + ) -> None: + + self._session.set_proxy_resolver(resolver) + + def create_request(self) -> HTTPRequest: + return HTTPRequest(self) + + +class HTTPRequest(GObject.GObject): + + __gtype_name__ = "HTTPRequest" + + __gsignals__ = { + 'starting-response-body': (SIGNAL_ACTIONS, None, ()), + 'response-progress': (SIGNAL_ACTIONS, None, (float,)), + 'request-progress': (SIGNAL_ACTIONS, None, (float,)), + 'finished': (SIGNAL_ACTIONS, None, ()), + 'destroy': (SIGNAL_ACTIONS, None, ()), + } + + def __init__(self, session: HTTPSession) -> None: + GObject.GObject.__init__(self) + + self._log = HTTPLogAdapter(log, extra={'request': self}) + + self._session = session + + self._received_size = 0 + self._sent_size = 0 + + self._cancellable = Gio.Cancellable() + self._input_stream = cast(Gio.InputStream, None) + self._output_stream: Optional[Gio.OutputStream] = None + self._is_finished = False + self._error: Optional[HTTPRequestError] = None + self._is_complete = False + self._timeout_reached = False + self._timeout_id = None + + self._response_body_file: Optional[Gio.File] = None + self._response_body_data = b'' + + self._request_body_file: Optional[Gio.File] = None + self._request_body_data: Optional[bytes] = None + + self._request_content_type = '' + self._request_content_length = 0 + + self._response_content_type = '' + self._response_content_length = 0 + + self._emit_request_progress = False + self._emit_response_progress = False + + self._message = Soup.Message() + self._user_data = None + + self._log.info('Created') + + def is_finished(self) -> bool: + return self._is_finished + + def is_complete(self) -> bool: + return self._is_complete + + def get_error(self) -> Optional[HTTPRequestError]: + return self._error + + def get_error_string(self) -> str: + if self._error == HTTPRequestError.STATUS_NOT_OK: + return self._message.get_reason_phrase() + return f'{self._error}' + + def get_response_headers(self) -> Soup.MessageHeaders: + return self._message.get_response_headers() + + def get_request_headers(self) -> Soup.MessageHeaders: + return self._message.get_request_headers() + + def get_data(self) -> bytes: + if not self._is_finished: + raise ValueError('Process not finished, data not available') + return self._response_body_data + + def get_uri(self) -> Optional[GLib.Uri]: + return self._message.get_uri() + + def get_status(self) -> Soup.Status: + return self._message.get_status() + + def get_soup_message(self) -> Soup.Message: + return self._message + + def set_user_data(self, user_data: Any) -> None: + self._user_data = user_data + + def get_user_data(self) -> Any: + return self._user_data + + def cancel(self) -> None: + if self._is_finished: + raise ValueError('Session already finished') + + self._cancellable.cancel() + + def set_request_body_from_path(self, content_type: str, path: Path) -> None: + if not path.exists(): + raise ValueError('%s does not exist' % path) + + if not path.is_file(): + raise ValueError('%s is not a file' % path) + + self._request_body_file = Gio.File.new_for_path(str(path)) + self._request_content_type = content_type + self._request_content_length = path.stat().st_size + + def set_request_body(self, content_type: str, data: bytes) -> None: + self._request_body_data = data + self._request_content_type = content_type + self._request_content_length = len(data) + + def set_response_body_from_path(self, path: Path) -> None: + self._response_body_file = Gio.File.new_for_path(str(path)) + + def connect(self, + signal_name: str, + callback: Any, + *args: Any + ) -> None: + + if signal_name == 'response-progress': + self._emit_response_progress = True + + if signal_name == 'request-progress': + self._emit_request_progress = True + + if signal_name in GObject.signal_list_names(HTTPRequest): + GObject.GObject.connect(self, signal_name, callback, *args) + return + + user_data = (callback, args) + + self._message.connect(signal_name, + self._on_connect_callback, + user_data) + + def _on_connect_callback(self, _message: Soup.Message, *args: Any) -> None: + signal_args = args[:-1] + callback, user_data = args[-1] + callback(self, *signal_args, *user_data) + + def send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None, + callback: Optional[Callable[[HTTPRequest], Any]] = None + ) -> None: + + if callback: + self.connect('finished', callback) + self._send(method, uri_string, timeout) + + def _send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None + ) -> None: + + if self._is_finished: + raise ValueError('Session already finished') + + self._message.set_method(method) + + if self._response_body_file is not None: + self._output_stream = self._response_body_file.replace( + None, + False, + Gio.FileCreateFlags.REPLACE_DESTINATION, + self._cancellable) + + uri = GLib.Uri.parse(uri_string, GLib.UriFlags.HAS_PASSWORD) + self._message.set_uri(uri) + + if self._request_body_data is not None: + self._message.set_request_body_from_bytes( + self._request_content_type, + GLib.Bytes.new(self._request_body_data) + ) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + if self._request_body_file is not None: + request_input_stream = self._request_body_file.read( + self._cancellable) + self._message.set_request_body(self._request_content_type, + request_input_stream, + self._request_content_length) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + self._message.connect('finished', self._on_finished) + + soup_session = self._session.get_soup_session() + soup_session.send_async(self._message, + GLib.PRIORITY_DEFAULT, + self._cancellable, + self._on_response) + + if timeout is not None: + self._timeout_id = GLib.timeout_add_seconds( + timeout, self._on_timeout) + + self._log.info('Request sent') + + def _on_request_body_progress(self, + _message: Soup.Message, + chunk: int) -> None: + self._sent_size =+ chunk + self.emit('request-progress', + self._sent_size / self._request_content_length) + + def _on_timeout(self) -> None: + self._timeout_reached = True + self.cancel() + + def _on_response(self, + session: Soup.Session, + result: Gio.AsyncResult + ) -> None: + + self._log.info('Request response received') + try: + self._input_stream = session.send_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._set_failed(HTTPRequestError.CANCELLED) + else: + self._set_failed(HTTPRequestError.UNKNOWN) + return + + if self._message.get_status() not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + headers = self.get_response_headers() + self._response_content_length = headers.get_content_length() + self._response_content_type, _params = headers.get_content_type() + if self._response_content_length == 0: + self._set_failed(HTTPRequestError.MISSING_CONTENT_LENGTH) + return + + self._log.info('Start downloading response body') + self.emit('starting-response-body') + + self._read_async() + + def _read_async(self) -> None: + self._input_stream.read_bytes_async(CHUNK_SIZE, + GLib.PRIORITY_LOW, + self._cancellable, + self._on_bytes_read_result) + + def _on_bytes_read_result(self, + input_stream: Gio.InputStream, + result: Gio.AsyncResult) -> None: + try: + data = input_stream.read_bytes_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + + return + + bytes_ = data.get_data() + if not bytes_: + self._finish_read() + return + + length = len(bytes_) + self._received_size += length + if self._received_size > self._response_content_length: + self._finish_read(HTTPRequestError.CONTENT_OVERFLOW) + return + + if self._output_stream is None: + self._response_body_data += bytes_ + + else: + try: + self._output_stream.write_all(bytes_, self._cancellable) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + return + + self._read_async() + + if (self._emit_response_progress and + self._message.get_method() == 'GET'): + self.emit('response-progress', + self._received_size / self._response_content_length) + + + def _finish_read(self, error: Optional[HTTPRequestError] = None) -> None: + self._log.info('Finished reading') + if error is None: + self._close_all_streams() + return + + self._set_failed(error) + + def _on_finished(self, _message: Soup.Message) -> None: + self._set_finished() + + def _set_finished(self) -> None: + status = self._message.get_status() + if status == Soup.Status.NONE: + # Message has not been sent, can happen when we cancel the message + # before it is sent. The finished signal triggers before the + # cancelled exception. + return + + if self._is_finished: + return + + headers = self._message.get_response_headers() + content_length = headers.get_content_length() + if self._received_size != content_length: + self._set_failed(HTTPRequestError.INCOMPLETE) + return + + if status not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + self._set_complete() + + def _set_failed(self, error: HTTPRequestError) -> None: + self._is_finished = True + if self._timeout_reached: + self._timeout_id = None + self._error = HTTPRequestError.TIMEOUT + else: + self._error = error + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _set_complete(self) -> None: + self._is_finished = True + self._is_complete = True + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _close_all_streams(self) -> None: + if self._input_stream is not None: + if not self._input_stream.is_closed(): + self._input_stream.close(None) + + if self._output_stream is not None: + if not self._output_stream.is_closed(): + self._output_stream.close(None) + + def _cleanup(self) -> None: + self._log.info('Run cleanup') + + self._response_body_data = b'' + self._message.run_dispose() + + del self._cancellable + del self._session + del self._input_stream + del self._output_stream + del self._user_data + + if self._timeout_id is not None: + GLib.source_remove(self._timeout_id) + + self.emit('destroy') + self.run_dispose() + + def __repr__(self) -> str: + return f'Request({id(self)})' ===================================== nbxmpp/modules/muclumbus.py ===================================== @@ -17,8 +17,6 @@ import json -from gi.repository import Soup - from nbxmpp.namespaces import Namespace from nbxmpp.protocol import Node from nbxmpp.protocol import Iq @@ -44,15 +42,6 @@ class Muclumbus(BaseModule): self._client = client self.handlers = [] - self._proxy_resolver = None - self._soup_session = Soup.Session() - - def set_proxy(self, proxy): - if proxy is None: - return - self._proxy_resolver = proxy.get_resolver() - self._soup_session.props.proxy_resolver = self._proxy_resolver - @iq_request_task def request_parameters(self, jid): task = yield @@ -141,24 +130,25 @@ class Muclumbus(BaseModule): if after is not None: search['after'] = after - message = Soup.Message.new('POST', uri) - message.set_request('application/json', - Soup.MemoryUse.COPY, - json.dumps(search).encode()) + body = json.dumps(search).encode() - response_message = yield message + session = self._client.http_session + request = session.create_request() + request.set_request_body('application/json', body) + request.send('POST', uri) - soup_body = response_message.get_property('response-body') + request = yield request - if response_message.status_code != 200: - self._log.warning(soup_body.data) + if not request.is_complete(): + self._log.warning(request.get_error_string()) yield MuclumbusResult(first=None, last=None, max=None, end=True, items=[]) - response = json.loads(soup_body.data) + response_body = request.get_data() + response = json.loads(response_body) result = response['result'] items = result.get('items') ===================================== nbxmpp/structs.py ===================================== @@ -30,8 +30,8 @@ from dataclasses import dataclass from dataclasses import field from datetime import datetime -from gi.repository import Soup from gi.repository import Gio +from gi.repository import GLib from nbxmpp.simplexml import Node from nbxmpp.namespaces import Namespace @@ -708,8 +708,9 @@ class ProxyData(NamedTuple): def get_uri(self) -> str: if self.username is not None: - user_pass = Soup.uri_encode('%s:%s' % (self.username, - self.password)) + username = GLib.uri_escape_string(self.username, None, False) + password = GLib.uri_escape_string(self.password, None, False) + user_pass = f'{username}:{password}' return '%s://%s@%s' % (self.type, user_pass, self.host) ===================================== nbxmpp/task.py ===================================== @@ -27,14 +27,14 @@ import logging from enum import IntEnum from functools import wraps -from gi.repository import Soup from gi.repository import GLib -from nbxmpp.modules.base import BaseModule from nbxmpp.simplexml import Node from nbxmpp.errors import is_error from nbxmpp.errors import CancelledError from nbxmpp.errors import TimeoutStanzaError +from nbxmpp.http import HTTPRequest +from nbxmpp.modules.base import BaseModule from nbxmpp.modules.util import make_func_arguments_string @@ -109,8 +109,7 @@ def http_request_task(func): @wraps(func) def func_wrapper(self, *args, callback=None, user_data=None, **kwargs): task = HTTPRequestTask(func(self, *args, **kwargs), - self._log, - self._soup_session) + self._log) return _setup_task(task, self._client, callback, user_data) return func_wrapper @@ -251,7 +250,7 @@ class Task: def _invoke_callbacks(self): for callback in self._done_callbacks: - if isinstance(callback, weakref.WeakMethod): + if isinstance(callback, (weakref.WeakMethod, weakref.ref)): callback = callback() if callback is None: return @@ -377,19 +376,21 @@ class HTTPRequestTask(Task): ''' - _process_types = (Soup.Message,) + _process_types = (HTTPRequest,) - def __init__(self, gen, logger, session): + def __init__(self, gen, logger): super().__init__(gen, logger) - self._session = session - def _run_async(self, message): + def _run_async(self, request: HTTPRequest): # pylint: disable=arguments-renamed - self._session.queue_message(message, self._async_finished, None) + request.connect('finished', self._async_finished) + + def _async_finished(self, request: HTTPRequest): + + if self._state == TaskState.CANCELLED: + return - def _async_finished(self, _session, message, _user_data): - if self._state != TaskState.CANCELLED: - self._next_step(message) + self._next_step(request) def _finalize(self): self._session = None ===================================== nbxmpp/websocket.py ===================================== @@ -22,7 +22,6 @@ from gi.repository import GLib from gi.repository import Gio from nbxmpp.const import TCPState -from nbxmpp.const import ConnectionType from nbxmpp.util import get_websocket_close_string from nbxmpp.util import convert_tls_error_flags from nbxmpp.connection import Connection @@ -35,11 +34,10 @@ class WebsocketConnection(Connection): Connection.__init__(self, *args, **kwargs) self._session = Soup.Session() - self._session.props.ssl_strict = False if self._log.getEffectiveLevel() == logging.INFO: self._session.add_feature( - Soup.Logger.new(Soup.LoggerLogLevel.BODY, -1)) + Soup.Logger.new(Soup.LoggerLogLevel.BODY)) self._websocket = None self._cancellable = Gio.Cancellable() @@ -53,11 +51,12 @@ class WebsocketConnection(Connection): self.state = TCPState.CONNECTING message = Soup.Message.new('GET', self._address.uri) - message.connect('starting', self._check_certificate) + message.connect('accept-certificate', self._check_certificate) message.set_flags(Soup.MessageFlags.NO_REDIRECT) self._session.websocket_connect_async(message, None, ['xmpp'], + GLib.PRIORITY_DEFAULT, self._cancellable, self._on_connect, None) @@ -91,19 +90,20 @@ class WebsocketConnection(Connection): # Soup.Session does this automatically raise NotImplementedError - def _check_certificate(self, message): - https_used, certificate, errors = message.get_https_status() - if not https_used and self._address.type == ConnectionType.PLAIN: - return + def _check_certificate(self, + _message, + certificate, + errors): self._peer_certificate = certificate self._peer_certificate_errors = convert_tls_error_flags(errors) if self._accept_certificate(): - return + return True self.notify('bad-certificate') self._cancellable.cancel() + return False def _on_websocket_message(self, _websocket, _type, message): data = message.get_data().decode() ===================================== setup.cfg ===================================== @@ -1,6 +1,6 @@ [metadata] name = nbxmpp -version = 3.2.5 +version = 4.0.0.dev1 description = XMPP Library author = Yann Leboulanger, Philipp Hoerist author_email = gajim-devel(a)gajim.org ===================================== test/unit/test_http.py ===================================== @@ -0,0 +1,246 @@ +from time import time +import tempfile +import unittest +from unittest.mock import call +from unittest.mock import Mock +from unittest.mock import patch +import os +from pathlib import Path + +from gi.repository import GLib +from gi.repository import Soup + +from nbxmpp.const import HTTPRequestError +from nbxmpp.http import HTTPSession + + +SMALL_FILE_URL = 'https://gajim.org/downloads/ci/unittest_small_file' +BIG_FILE_URL = 'https://gajim.org/downloads/ci/unittest_big_file' +NO_FILE_URL = 'https://gajim.org/downloads/ci/no-file' + + +# import logging +# consoleloghandler = logging.StreamHandler() +# log = logging.getLogger('nbxmpp.http') +# log.setLevel(logging.DEBUG) +# log.addHandler(consoleloghandler) + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class HTTP(unittest.TestCase): + + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def test_download_to_file(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + + temp_dir = tempfile.gettempdir() + request.set_response_body_from_path(Path(temp_dir) / 'file') + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_in_memory_download(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_before_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + GLib.timeout_add(10, request.cancel) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.progress.assert_not_called() + callback_mock.starting.assert_not_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_after_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + def _on_progress(req, progress): + self.assertIsInstance(progress, float) + req.cancel() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('response-progress', _on_progress) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', BIG_FILE_URL, timeout=10) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_failed_404(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('finished', callback_mock.finished) + request.connect('response-progress', callback_mock.progress) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', NO_FILE_URL, timeout=5) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.STATUS_NOT_OK) + self.assertEqual(request.get_status(), Soup.Status.NOT_FOUND) + + callback_mock.progress.assert_not_called() + callback_mock.finished.assert_called() + + def test_cancel_with_timeout(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', BIG_FILE_URL, timeout=1) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertFalse(request.is_complete()) + self.assertEqual(request.get_error(), HTTPRequestError.TIMEOUT) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_signal_propagation(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting', callback_mock.starting, 1, 2, 3) + request.connect('got-body', callback_mock.got_body) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertTrue(request.is_complete()) + + callback_mock.starting.assert_called_with(request, 1, 2, 3) + callback_mock.got_body.assert_called_with(request) + + def test_parallel_download(self): + + mainloop = GLib.MainLoop() + + request1 = HTTPSession().create_request() + request1.send('GET', SMALL_FILE_URL, timeout=5) + + request2 = HTTPSession().create_request() + request2.send('GET', SMALL_FILE_URL, timeout=5) + + request3 = HTTPSession().create_request() + request3.send('GET', SMALL_FILE_URL, timeout=5) + + request4 = HTTPSession().create_request() + request4.send('GET', SMALL_FILE_URL, timeout=5) + + GLib.timeout_add_seconds(5, mainloop.quit) + + mainloop.run() + + self.assertTrue(request1.is_finished()) + self.assertTrue(request1.is_complete()) + + self.assertTrue(request2.is_finished()) + self.assertTrue(request2.is_complete()) + + self.assertTrue(request3.is_finished()) + self.assertTrue(request3.is_complete()) + + self.assertTrue(request4.is_finished()) + self.assertTrue(request4.is_complete()) + + +if __name__ == '__main__': + unittest.main() ===================================== test/unit/test_muclumbus.py ===================================== @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Any + +from gi.repository import GLib + +import os +import unittest +from unittest.mock import Mock +from nbxmpp.client import Client +from nbxmpp.http import HTTPSession +from nbxmpp.structs import MuclumbusResult +from nbxmpp.modules.muclumbus import Muclumbus + +from nbxmpp.sasl import SCRAM_SHA_1 +from nbxmpp.util import b64encode + +# Test vector from https://wiki.xmpp.org/web/SASL_and_SCRAM-SHA-1 + +API_URL = 'https://search.jabber.network/api/1.0/search' + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class TestMuclumbus(unittest.TestCase): + def setUp(self): + self._client = Client() + self._client.set_http_session(HTTPSession()) + self._module = Muclumbus(self._client) + + def test_http_request(self): + + mainloop = GLib.MainLoop() + + def _result(task: Any) -> None: + result = task.finish() + assert isinstance(result, MuclumbusResult) + assert len(result.items) > 0 + mainloop.quit() + + self._module.set_http_search(API_URL, 'gajim', callback=_result) + + mainloop.run() + +if __name__ == '__main__': + unittest.main() View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/compare/2726ebaf4611c7f04d432f8… -- View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/compare/2726ebaf4611c7f04d432f8… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/python-nbxmpp][soup3] change: Port to libsoup3
by Philipp Hörist (＠lovetox) 27 Dec '22

27 Dec '22

Philipp Hörist pushed to branch soup3 at gajim / python-nbxmpp Commits: 2726ebaf by Philipp Hörist at 2022-12-27T22:42:24+01:00 change: Port to libsoup3 - Port code to libsoup3 - Wrap Soup.Session and Soup.Message for easier usage - - - - - 15 changed files: - .gitlab-ci.yml - README.md - debian/control - nbxmpp/__init__.py - nbxmpp/addresses.py - nbxmpp/client.py - nbxmpp/const.py - nbxmpp/examples/client.py - + nbxmpp/http.py - nbxmpp/modules/muclumbus.py - nbxmpp/structs.py - nbxmpp/task.py - nbxmpp/websocket.py - + test/unit/test_http.py - + test/unit/test_muclumbus.py Changes: ===================================== .gitlab-ci.yml ===================================== @@ -9,7 +9,7 @@ test-pylint: stage: test script: - python3 .ci/pylint-test.py - - coverage run --source=nbxmpp -m unittest discover -v + - NBXMPP_EXTERNAL_UNIT_TESTS=1 coverage run --source=nbxmpp -m unittest discover -v - coverage report -mi - coverage xml -i - pip install . ===================================== README.md ===================================== @@ -7,7 +7,7 @@ - python >= 3.9.0 - PyGObject - GLib >= 2.60 -- libsoup +- libsoup3 - precis-i18n - packaging - idna ===================================== debian/control ===================================== @@ -6,7 +6,7 @@ Build-Depends: debhelper-compat (= 13), dh-python, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-all, python3-gi, python3-idna, @@ -24,7 +24,7 @@ Architecture: all Depends: ${misc:Depends}, ${python3:Depends}, libglib2.0-0 (>= 2.60), - gir1.2-soup-2.4, + gir1.2-soup-3.0, python3-gi, python3-idna, python3-packaging, ===================================== nbxmpp/__init__.py ===================================== @@ -2,6 +2,6 @@ import gi from .protocol import * -gi.require_version('Soup', '2.4') +gi.require_version('Soup', '3.0') __version__: str = '3.2.5' ===================================== nbxmpp/addresses.py ===================================== @@ -18,9 +18,8 @@ import logging from collections import namedtuple -from gi.repository import Soup - -import nbxmpp +from nbxmpp.http import HTTPRequest +from nbxmpp.http import HTTPSession from nbxmpp.util import Observable from nbxmpp.const import ConnectionType from nbxmpp.const import ConnectionProtocol @@ -63,6 +62,7 @@ class ServerAddresses(Observable): Observable.__init__(self, log) self._domain = domain + self._http_session = None self._custom_host = None self._proxy = None self._is_resolved = False @@ -136,32 +136,31 @@ class ServerAddresses(Observable): self._resolve_alternatives() def _resolve_alternatives(self) -> None: - session = Soup.Session() - session.props.timeout = 5 - session.props.user_agent = f'nbxmpp/{nbxmpp.__version__}' - message = Soup.Message.new( - 'GET', f'https://{self._domain}/.well-known/host-meta') - session.queue_message(message, self._on_alternatives_result) - - def _on_alternatives_result(self, - _session: Soup.Session, - message: Soup.Message) -> None: - - status = message.props.status_code - if status != Soup.Status.OK: - error = message.props.reason_phrase - log.info('Failed to retrieve host-meta file: %s %s', status, error) + if self._http_session is None: self._on_request_resolved() return - response_body = message.props.response_body - if response_body is None or not response_body.data: + request = self._http_session.create_request() + request.send('GET', f'https://{self._domain}/.well-known/host-meta', + timeout=5, + callback=self._on_alternatives_result) + + def _on_alternatives_result(self, request: HTTPRequest) -> None: + + if not request.is_complete(): + log.info('Failed to retrieve host-meta file: %s', + request.get_error_string()) + self._on_request_resolved() + return + + response_body = request.get_data() + if not response_body: log.info('No response body data found') self._on_request_resolved() return try: - uri = parse_websocket_uri(response_body.data) + uri = parse_websocket_uri(response_body.decode()) except Exception as error: log.info('Error parsing websocket uri: %s', error) self._on_request_resolved() @@ -215,6 +214,9 @@ class ServerAddresses(Observable): def set_proxy(self, proxy): self._proxy = proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + def _on_request_resolved(self): self._is_resolved = True self.notify('resolved') ===================================== nbxmpp/client.py ===================================== @@ -22,6 +22,7 @@ from typing import Optional from gi.repository import GLib from nbxmpp.namespaces import Namespace +from nbxmpp.http import HTTPSession from nbxmpp.protocol import Features from nbxmpp.protocol import StanzaMalformed from nbxmpp.protocol import SessionRequest @@ -93,6 +94,7 @@ class Client(Observable): self._client_cert = None self._client_cert_pass = None self._proxy = None + self._http_session = HTTPSession() self._allowed_con_types = None self._allowed_protocols = None @@ -329,12 +331,18 @@ class Client(Observable): def set_proxy(self, proxy): self._proxy = proxy - self._dispatcher.get_module('Muclumbus').set_proxy(proxy) @property def proxy(self): return self._proxy + def set_http_session(self, session: HTTPSession) -> None: + self._http_session = session + + @property + def http_session(self) -> HTTPSession: + return self._http_session + def get_bound_jid(self): return self._jid @@ -402,6 +410,7 @@ class Client(Observable): self.state = StreamState.RESOLVE self._addresses = ServerAddresses(self._domain) + self._addresses.set_http_session(self._http_session) self._addresses.set_custom_host(self._custom_host) self._addresses.set_proxy(self._proxy) self._addresses.subscribe('resolved', self._on_addresses_resolved) ===================================== nbxmpp/const.py ===================================== @@ -453,6 +453,16 @@ class Mode(IntEnum): return self == Mode.ANONYMOUS_TEST +class HTTPRequestError(IntEnum): + UNKNOWN = 0 + MISSING_CONTENT_LENGTH = 1 + INCOMPLETE = 2 + STATUS_NOT_OK = 3 + CANCELLED = 4 + CONTENT_OVERFLOW = 5 + TIMEOUT = 6 + + MOODS = [ 'afraid', 'amazed', ===================================== nbxmpp/examples/client.py ===================================== @@ -6,6 +6,7 @@ import json from pathlib import Path import gi + gi.require_version('Gtk', '3.0') gi.require_version('GLib', '2.0') from gi.repository import Gtk @@ -14,6 +15,7 @@ from gi.repository import GLib import nbxmpp from nbxmpp.protocol import JID from nbxmpp.client import Client +from nbxmpp.http import HTTPSession from nbxmpp.structs import ProxyData from nbxmpp.structs import StanzaHandler from nbxmpp.const import ConnectionType @@ -83,6 +85,7 @@ class TestClient(Gtk.Window): self._client.set_domain(self.address.domain) self._client.set_username(self.address.localpart) self._client.set_resource('test') + self._client.set_http_session(HTTPSession()) proxy_ip = self._builder.proxy_ip.get_text() if proxy_ip: ===================================== nbxmpp/http.py ===================================== @@ -0,0 +1,473 @@ +# This file is part of nbxmpp. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; If not, see <http://www.gnu.org/licenses/>. + +from __future__ import annotations + +from pathlib import Path + +from typing import Any +from typing import cast +from typing import Literal +from typing import Callable +from typing import Optional + +import logging + +from gi.repository import Soup +from gi.repository import Gio +from gi.repository import GLib +from gi.repository import GObject + +import nbxmpp +from .const import HTTPRequestError + + +log = logging.getLogger('nbxmpp.http') + + +HTTP_METHODS_T = Literal[ + 'CONNECT', + 'DELETE', + 'GET', + 'HEAD', + 'OPTIONS', + 'PATCH', + 'POST', + 'PUT', + 'TRACE', +] +CHUNK_SIZE = 32768 +DEFAULT_USER_AGENT = f'nbxmpp/{nbxmpp.__version__}' +SIGNAL_ACTIONS = GObject.SignalFlags.RUN_LAST | GObject.SignalFlags.ACTION + + +class HTTPLogAdapter(logging.LoggerAdapter): + def process(self, msg: str, kwargs: Any) -> tuple[str, Any]: + return f'{self.extra["request"]}: {msg}', kwargs + + +class HTTPSession: + def __init__(self, + user_agent: str = DEFAULT_USER_AGENT, + sniffer: bool = False + ) -> None: + + self._session = Soup.Session() + self._session.set_user_agent(user_agent) + + if sniffer: + self._session.add_feature_by_type(Soup.ContentSniffer) + + def get_soup_session(self) -> Soup.Session: + return self._session + + def set_proxy_resolver(self, + resolver: Optional[Gio.SimpleProxyResolver] + ) -> None: + + self._session.set_proxy_resolver(resolver) + + def create_request(self) -> HTTPRequest: + return HTTPRequest(self) + + +class HTTPRequest(GObject.GObject): + + __gtype_name__ = "HTTPRequest" + + __gsignals__ = { + 'starting-response-body': (SIGNAL_ACTIONS, None, ()), + 'response-progress': (SIGNAL_ACTIONS, None, (float,)), + 'request-progress': (SIGNAL_ACTIONS, None, (float,)), + 'finished': (SIGNAL_ACTIONS, None, ()), + 'destroy': (SIGNAL_ACTIONS, None, ()), + } + + def __init__(self, session: HTTPSession) -> None: + GObject.GObject.__init__(self) + + self._log = HTTPLogAdapter(log, extra={'request': self}) + + self._session = session + + self._received_size = 0 + self._sent_size = 0 + + self._cancellable = Gio.Cancellable() + self._input_stream = cast(Gio.InputStream, None) + self._output_stream: Optional[Gio.OutputStream] = None + self._is_finished = False + self._error: Optional[HTTPRequestError] = None + self._is_complete = False + self._timeout_reached = False + self._timeout_id = None + + self._response_body_file: Optional[Gio.File] = None + self._response_body_data = b'' + + self._request_body_file: Optional[Gio.File] = None + self._request_body_data: Optional[bytes] = None + + self._request_content_type = '' + self._request_content_length = 0 + + self._response_content_type = '' + self._response_content_length = 0 + + self._emit_request_progress = False + self._emit_response_progress = False + + self._message = Soup.Message() + self._user_data = None + + self._log.info('Created') + + def is_finished(self) -> bool: + return self._is_finished + + def is_complete(self) -> bool: + return self._is_complete + + def get_error(self) -> Optional[HTTPRequestError]: + return self._error + + def get_error_string(self) -> str: + if self._error == HTTPRequestError.STATUS_NOT_OK: + return self._message.get_reason_phrase() + return f'{self._error}' + + def get_response_headers(self) -> Soup.MessageHeaders: + return self._message.get_response_headers() + + def get_request_headers(self) -> Soup.MessageHeaders: + return self._message.get_request_headers() + + def get_data(self) -> bytes: + if not self._is_finished: + raise ValueError('Process not finished, data not available') + return self._response_body_data + + def get_uri(self) -> Optional[GLib.Uri]: + return self._message.get_uri() + + def get_status(self) -> Soup.Status: + return self._message.get_status() + + def get_soup_message(self) -> Soup.Message: + return self._message + + def set_user_data(self, user_data: Any) -> None: + self._user_data = user_data + + def get_user_data(self) -> Any: + return self._user_data + + def cancel(self) -> None: + if self._is_finished: + raise ValueError('Session already finished') + + self._cancellable.cancel() + + def set_request_body_from_path(self, content_type: str, path: Path) -> None: + if not path.exists(): + raise ValueError('%s does not exist' % path) + + if not path.is_file(): + raise ValueError('%s is not a file' % path) + + self._request_body_file = Gio.File.new_for_path(str(path)) + self._request_content_type = content_type + self._request_content_length = path.stat().st_size + + def set_request_body(self, content_type: str, data: bytes) -> None: + self._request_body_data = data + self._request_content_type = content_type + self._request_content_length = len(data) + + def set_response_body_from_path(self, path: Path) -> None: + self._response_body_file = Gio.File.new_for_path(str(path)) + + def connect(self, + signal_name: str, + callback: Any, + *args: Any + ) -> None: + + if signal_name == 'response-progress': + self._emit_response_progress = True + + if signal_name == 'request-progress': + self._emit_request_progress = True + + if signal_name in GObject.signal_list_names(HTTPRequest): + GObject.GObject.connect(self, signal_name, callback, *args) + return + + user_data = (callback, args) + + self._message.connect(signal_name, + self._on_connect_callback, + user_data) + + def _on_connect_callback(self, _message: Soup.Message, *args: Any) -> None: + signal_args = args[:-1] + callback, user_data = args[-1] + callback(self, *signal_args, *user_data) + + def send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None, + callback: Optional[Callable[[HTTPRequest], Any]] = None + ) -> None: + + if callback: + self.connect('finished', callback) + self._send(method, uri_string, timeout) + + def _send(self, + method: HTTP_METHODS_T, + uri_string: str, + timeout: Optional[int] = None + ) -> None: + + if self._is_finished: + raise ValueError('Session already finished') + + self._message.set_method(method) + + if self._response_body_file is not None: + self._output_stream = self._response_body_file.replace( + None, + False, + Gio.FileCreateFlags.REPLACE_DESTINATION, + self._cancellable) + + uri = GLib.Uri.parse(uri_string, GLib.UriFlags.HAS_PASSWORD) + self._message.set_uri(uri) + + if self._request_body_data is not None: + self._message.set_request_body_from_bytes( + self._request_content_type, + GLib.Bytes.new(self._request_body_data) + ) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + if self._request_body_file is not None: + request_input_stream = self._request_body_file.read( + self._cancellable) + self._message.set_request_body(self._request_content_type, + request_input_stream, + self._request_content_length) + if self._emit_request_progress: + self._message.connect('wrote-body-data', + self._on_request_body_progress) + + self._message.connect('finished', self._on_finished) + + soup_session = self._session.get_soup_session() + soup_session.send_async(self._message, + GLib.PRIORITY_DEFAULT, + self._cancellable, + self._on_response) + + if timeout is not None: + self._timeout_id = GLib.timeout_add_seconds( + timeout, self._on_timeout) + + self._log.info('Request sent') + + def _on_request_body_progress(self, + _message: Soup.Message, + chunk: int) -> None: + self._sent_size =+ chunk + self.emit('request-progress', + self._sent_size / self._request_content_length) + + def _on_timeout(self) -> None: + self._timeout_reached = True + self.cancel() + + def _on_response(self, + session: Soup.Session, + result: Gio.AsyncResult + ) -> None: + + self._log.info('Request response received') + try: + self._input_stream = session.send_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._set_failed(HTTPRequestError.CANCELLED) + else: + self._set_failed(HTTPRequestError.UNKNOWN) + return + + if self._message.get_status() not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + headers = self.get_response_headers() + self._response_content_length = headers.get_content_length() + self._response_content_type, _params = headers.get_content_type() + if self._response_content_length == 0: + self._set_failed(HTTPRequestError.MISSING_CONTENT_LENGTH) + return + + self._log.info('Start downloading response body') + self.emit('starting-response-body') + + self._read_async() + + def _read_async(self) -> None: + self._input_stream.read_bytes_async(CHUNK_SIZE, + GLib.PRIORITY_LOW, + self._cancellable, + self._on_bytes_read_result) + + def _on_bytes_read_result(self, + input_stream: Gio.InputStream, + result: Gio.AsyncResult) -> None: + try: + data = input_stream.read_bytes_finish(result) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + + return + + bytes_ = data.get_data() + if not bytes_: + self._finish_read() + return + + length = len(bytes_) + self._received_size += length + if self._received_size > self._response_content_length: + self._finish_read(HTTPRequestError.CONTENT_OVERFLOW) + return + + if self._output_stream is None: + self._response_body_data += bytes_ + + else: + try: + self._output_stream.write_all(bytes_, self._cancellable) + except GLib.Error as error: + self._log.error(error) + quark = GLib.quark_try_string('g-io-error-quark') + if error.matches(quark, Gio.IOErrorEnum.CANCELLED): + self._finish_read(HTTPRequestError.CANCELLED) + else: + self._finish_read(HTTPRequestError.UNKNOWN) + return + + self._read_async() + + if (self._emit_response_progress and + self._message.get_method() == 'GET'): + self.emit('response-progress', + self._received_size / self._response_content_length) + + + def _finish_read(self, error: Optional[HTTPRequestError] = None) -> None: + self._log.info('Finished reading') + if error is None: + self._close_all_streams() + return + + self._set_failed(error) + + def _on_finished(self, _message: Soup.Message) -> None: + self._set_finished() + + def _set_finished(self) -> None: + status = self._message.get_status() + if status == Soup.Status.NONE: + # Message has not been sent, can happen when we cancel the message + # before it is sent. The finished signal triggers before the + # cancelled exception. + return + + if self._is_finished: + return + + headers = self._message.get_response_headers() + content_length = headers.get_content_length() + if self._received_size != content_length: + self._set_failed(HTTPRequestError.INCOMPLETE) + return + + if status not in (Soup.Status.OK, Soup.Status.CREATED): + self._set_failed(HTTPRequestError.STATUS_NOT_OK) + return + + self._set_complete() + + def _set_failed(self, error: HTTPRequestError) -> None: + self._is_finished = True + if self._timeout_reached: + self._timeout_id = None + self._error = HTTPRequestError.TIMEOUT + else: + self._error = error + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _set_complete(self) -> None: + self._is_finished = True + self._is_complete = True + self._close_all_streams() + self.emit('finished') + self._cleanup() + + def _close_all_streams(self) -> None: + if self._input_stream is not None: + if not self._input_stream.is_closed(): + self._input_stream.close(None) + + if self._output_stream is not None: + if not self._output_stream.is_closed(): + self._output_stream.close(None) + + def _cleanup(self) -> None: + self._log.info('Run cleanup') + + self._response_body_data = b'' + self._message.run_dispose() + + del self._cancellable + del self._session + del self._input_stream + del self._output_stream + del self._user_data + + if self._timeout_id is not None: + GLib.source_remove(self._timeout_id) + + self.emit('destroy') + self.run_dispose() + + def __repr__(self) -> str: + return f'Request({id(self)})' ===================================== nbxmpp/modules/muclumbus.py ===================================== @@ -17,8 +17,6 @@ import json -from gi.repository import Soup - from nbxmpp.namespaces import Namespace from nbxmpp.protocol import Node from nbxmpp.protocol import Iq @@ -44,15 +42,6 @@ class Muclumbus(BaseModule): self._client = client self.handlers = [] - self._proxy_resolver = None - self._soup_session = Soup.Session() - - def set_proxy(self, proxy): - if proxy is None: - return - self._proxy_resolver = proxy.get_resolver() - self._soup_session.props.proxy_resolver = self._proxy_resolver - @iq_request_task def request_parameters(self, jid): task = yield @@ -141,24 +130,25 @@ class Muclumbus(BaseModule): if after is not None: search['after'] = after - message = Soup.Message.new('POST', uri) - message.set_request('application/json', - Soup.MemoryUse.COPY, - json.dumps(search).encode()) + body = json.dumps(search).encode() - response_message = yield message + session = self._client.http_session + request = session.create_request() + request.set_request_body('application/json', body) + request.send('POST', uri) - soup_body = response_message.get_property('response-body') + request = yield request - if response_message.status_code != 200: - self._log.warning(soup_body.data) + if not request.is_complete(): + self._log.warning(request.get_error_string()) yield MuclumbusResult(first=None, last=None, max=None, end=True, items=[]) - response = json.loads(soup_body.data) + response_body = request.get_data() + response = json.loads(response_body) result = response['result'] items = result.get('items') ===================================== nbxmpp/structs.py ===================================== @@ -30,8 +30,8 @@ from dataclasses import dataclass from dataclasses import field from datetime import datetime -from gi.repository import Soup from gi.repository import Gio +from gi.repository import GLib from nbxmpp.simplexml import Node from nbxmpp.namespaces import Namespace @@ -708,8 +708,9 @@ class ProxyData(NamedTuple): def get_uri(self) -> str: if self.username is not None: - user_pass = Soup.uri_encode('%s:%s' % (self.username, - self.password)) + username = GLib.uri_escape_string(self.username, None, False) + password = GLib.uri_escape_string(self.password, None, False) + user_pass = f'{username}:{password}' return '%s://%s@%s' % (self.type, user_pass, self.host) ===================================== nbxmpp/task.py ===================================== @@ -27,14 +27,14 @@ import logging from enum import IntEnum from functools import wraps -from gi.repository import Soup from gi.repository import GLib -from nbxmpp.modules.base import BaseModule from nbxmpp.simplexml import Node from nbxmpp.errors import is_error from nbxmpp.errors import CancelledError from nbxmpp.errors import TimeoutStanzaError +from nbxmpp.http import HTTPRequest +from nbxmpp.modules.base import BaseModule from nbxmpp.modules.util import make_func_arguments_string @@ -109,8 +109,7 @@ def http_request_task(func): @wraps(func) def func_wrapper(self, *args, callback=None, user_data=None, **kwargs): task = HTTPRequestTask(func(self, *args, **kwargs), - self._log, - self._soup_session) + self._log) return _setup_task(task, self._client, callback, user_data) return func_wrapper @@ -251,7 +250,7 @@ class Task: def _invoke_callbacks(self): for callback in self._done_callbacks: - if isinstance(callback, weakref.WeakMethod): + if isinstance(callback, (weakref.WeakMethod, weakref.ref)): callback = callback() if callback is None: return @@ -377,19 +376,21 @@ class HTTPRequestTask(Task): ''' - _process_types = (Soup.Message,) + _process_types = (HTTPRequest,) - def __init__(self, gen, logger, session): + def __init__(self, gen, logger): super().__init__(gen, logger) - self._session = session - def _run_async(self, message): + def _run_async(self, request: HTTPRequest): # pylint: disable=arguments-renamed - self._session.queue_message(message, self._async_finished, None) + request.connect('finished', self._async_finished) + + def _async_finished(self, request: HTTPRequest): + + if self._state == TaskState.CANCELLED: + return - def _async_finished(self, _session, message, _user_data): - if self._state != TaskState.CANCELLED: - self._next_step(message) + self._next_step(request) def _finalize(self): self._session = None ===================================== nbxmpp/websocket.py ===================================== @@ -22,7 +22,6 @@ from gi.repository import GLib from gi.repository import Gio from nbxmpp.const import TCPState -from nbxmpp.const import ConnectionType from nbxmpp.util import get_websocket_close_string from nbxmpp.util import convert_tls_error_flags from nbxmpp.connection import Connection @@ -35,11 +34,10 @@ class WebsocketConnection(Connection): Connection.__init__(self, *args, **kwargs) self._session = Soup.Session() - self._session.props.ssl_strict = False if self._log.getEffectiveLevel() == logging.INFO: self._session.add_feature( - Soup.Logger.new(Soup.LoggerLogLevel.BODY, -1)) + Soup.Logger.new(Soup.LoggerLogLevel.BODY)) self._websocket = None self._cancellable = Gio.Cancellable() @@ -53,11 +51,12 @@ class WebsocketConnection(Connection): self.state = TCPState.CONNECTING message = Soup.Message.new('GET', self._address.uri) - message.connect('starting', self._check_certificate) + message.connect('accept-certificate', self._check_certificate) message.set_flags(Soup.MessageFlags.NO_REDIRECT) self._session.websocket_connect_async(message, None, ['xmpp'], + GLib.PRIORITY_DEFAULT, self._cancellable, self._on_connect, None) @@ -91,19 +90,20 @@ class WebsocketConnection(Connection): # Soup.Session does this automatically raise NotImplementedError - def _check_certificate(self, message): - https_used, certificate, errors = message.get_https_status() - if not https_used and self._address.type == ConnectionType.PLAIN: - return + def _check_certificate(self, + _message, + certificate, + errors): self._peer_certificate = certificate self._peer_certificate_errors = convert_tls_error_flags(errors) if self._accept_certificate(): - return + return True self.notify('bad-certificate') self._cancellable.cancel() + return False def _on_websocket_message(self, _websocket, _type, message): data = message.get_data().decode() ===================================== test/unit/test_http.py ===================================== @@ -0,0 +1,246 @@ +from time import time +import tempfile +import unittest +from unittest.mock import call +from unittest.mock import Mock +from unittest.mock import patch +import os +from pathlib import Path + +from gi.repository import GLib +from gi.repository import Soup + +from nbxmpp.const import HTTPRequestError +from nbxmpp.http import HTTPSession + + +SMALL_FILE_URL = 'https://gajim.org/downloads/ci/unittest_small_file' +BIG_FILE_URL = 'https://gajim.org/downloads/ci/unittest_big_file' +NO_FILE_URL = 'https://gajim.org/downloads/ci/no-file' + + +# import logging +# consoleloghandler = logging.StreamHandler() +# log = logging.getLogger('nbxmpp.http') +# log.setLevel(logging.DEBUG) +# log.addHandler(consoleloghandler) + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class HTTP(unittest.TestCase): + + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def test_download_to_file(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + + temp_dir = tempfile.gettempdir() + request.set_response_body_from_path(Path(temp_dir) / 'file') + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_in_memory_download(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertIsNone(request.get_error()) + + callback_mock.progress.assert_called() + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_before_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('response-progress', callback_mock.progress) + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + GLib.timeout_add(10, request.cancel) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.progress.assert_not_called() + callback_mock.starting.assert_not_called() + callback_mock.finished.assert_called() + + def test_download_cancelled_after_start(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + def _on_progress(req, progress): + self.assertIsInstance(progress, float) + req.cancel() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('response-progress', _on_progress) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', BIG_FILE_URL, timeout=10) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.CANCELLED) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_download_failed_404(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('finished', callback_mock.finished) + request.connect('response-progress', callback_mock.progress) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', NO_FILE_URL, timeout=5) + + mainloop.run() + + self.assertFalse(request.is_complete()) + self.assertTrue(request.is_finished()) + self.assertEqual(request.get_error(), HTTPRequestError.STATUS_NOT_OK) + self.assertEqual(request.get_status(), Soup.Status.NOT_FOUND) + + callback_mock.progress.assert_not_called() + callback_mock.finished.assert_called() + + def test_cancel_with_timeout(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting-response-body', callback_mock.starting) + request.connect('finished', callback_mock.finished) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', BIG_FILE_URL, timeout=1) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertFalse(request.is_complete()) + self.assertEqual(request.get_error(), HTTPRequestError.TIMEOUT) + + callback_mock.starting.assert_called() + callback_mock.finished.assert_called() + + def test_signal_propagation(self): + + mainloop = GLib.MainLoop() + + session = HTTPSession() + request = session.create_request() + + callback_mock = Mock() + request.connect('starting', callback_mock.starting, 1, 2, 3) + request.connect('got-body', callback_mock.got_body) + request.connect('destroy', lambda *args: mainloop.quit()) + request.send('GET', SMALL_FILE_URL, timeout=10) + + mainloop.run() + + self.assertTrue(request.is_finished()) + self.assertTrue(request.is_complete()) + + callback_mock.starting.assert_called_with(request, 1, 2, 3) + callback_mock.got_body.assert_called_with(request) + + def test_parallel_download(self): + + mainloop = GLib.MainLoop() + + request1 = HTTPSession().create_request() + request1.send('GET', SMALL_FILE_URL, timeout=5) + + request2 = HTTPSession().create_request() + request2.send('GET', SMALL_FILE_URL, timeout=5) + + request3 = HTTPSession().create_request() + request3.send('GET', SMALL_FILE_URL, timeout=5) + + request4 = HTTPSession().create_request() + request4.send('GET', SMALL_FILE_URL, timeout=5) + + GLib.timeout_add_seconds(5, mainloop.quit) + + mainloop.run() + + self.assertTrue(request1.is_finished()) + self.assertTrue(request1.is_complete()) + + self.assertTrue(request2.is_finished()) + self.assertTrue(request2.is_complete()) + + self.assertTrue(request3.is_finished()) + self.assertTrue(request3.is_complete()) + + self.assertTrue(request4.is_finished()) + self.assertTrue(request4.is_complete()) + + +if __name__ == '__main__': + unittest.main() ===================================== test/unit/test_muclumbus.py ===================================== @@ -0,0 +1,46 @@ +from __future__ import annotations + +from typing import Any + +from gi.repository import GLib + +import os +import unittest +from unittest.mock import Mock +from nbxmpp.client import Client +from nbxmpp.http import HTTPSession +from nbxmpp.structs import MuclumbusResult +from nbxmpp.modules.muclumbus import Muclumbus + +from nbxmpp.sasl import SCRAM_SHA_1 +from nbxmpp.util import b64encode + +# Test vector from https://wiki.xmpp.org/web/SASL_and_SCRAM-SHA-1 + +API_URL = 'https://search.jabber.network/api/1.0/search' + + +(a)unittest.skipUnless(os.environ.get('NBXMPP_EXTERNAL_UNIT_TESTS'), + 'ENV var for external tests not set') +class TestMuclumbus(unittest.TestCase): + def setUp(self): + self._client = Client() + self._client.set_http_session(HTTPSession()) + self._module = Muclumbus(self._client) + + def test_http_request(self): + + mainloop = GLib.MainLoop() + + def _result(task: Any) -> None: + result = task.finish() + assert isinstance(result, MuclumbusResult) + assert len(result.items) > 0 + mainloop.quit() + + self._module.set_http_search(API_URL, 'gajim', callback=_result) + + mainloop.run() + +if __name__ == '__main__': + unittest.main() View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/2726ebaf4611c7f04d432f8f… -- View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/2726ebaf4611c7f04d432f8f… You're receiving this email because of your account on dev.gajim.org.

1 0

Jump to page:

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

Commits