Philipp Hörist pushed to branch master at gajim / gajim

Commits: b84ef377 by mjk at 2022-11-30T21:05:07+00:00 imprv: Styling: Introduce URI scheme whitelisting

Fixes #11266

Co-authored-by: Philipp Hörist <philipp@hoerist.com>

- - - - - ec0008b7 by mjk at 2022-11-30T21:17:13+00:00 fix: Styling: Linkify URIs only if at the start of a word

- - - - - 88989297 by mjk at 2022-11-30T21:19:03+00:00 refactor: Tests: Improve failure reports in test_styling

- - - - -

8 changed files:

- .gitlab/issue_templates/Release.md - gajim/common/const.py - gajim/common/helpers.py - + gajim/common/iana.py - gajim/common/setting_values.py - gajim/common/styling.py - + scripts/get_iana_data.py - test/no_gui/test_styling.py

Changes:

===================================== .gitlab/issue_templates/Release.md ===================================== @@ -2,6 +2,7 @@

## Build * [ ] Merge translations from Weblate +* [ ] Update IANA data with `./scripts/get_iana_data.py gajim/common/iana.py` * [ ] Run `./scripts/bump_version.py x.x.x` * [ ] Push release tag `x.x.x`

===================================== gajim/common/const.py ===================================== @@ -944,97 +944,6 @@ def is_active(self): GAJIM_WIKI_URI = 'https://dev.gajim.org/gajim/gajim/wikis' GAJIM_SUPPORT_JID = 'gajim@conference.gajim.org'

-URI_SCHEMES = { - 'aaa://', - 'aaas://', - 'acap://', - 'cap://', - 'cid:', - 'crid://', - 'data:', - 'dav:', - 'dict://', - 'dns:', - 'fax:', - 'file:/', - 'ftp://', - 'geo:', - 'go:', - 'gopher://', - 'h323:', - 'http://', - 'https://', - 'iax:', - 'icap://', - 'im:', - 'imap://', - 'info:', - 'ipp://', - 'iris:', - 'iris.beep:', - 'iris.xpc:', - 'iris.xpcs:', - 'iris.lwz:', - 'ldap://', - 'mailto:', - 'mid:', - 'modem:', - 'msrp://', - 'msrps://', - 'mtqp://', - 'mupdate://', - 'news:', - 'nfs://', - 'nntp://', - 'opaquelocktoken:', - 'pop://', - 'pres:', - 'prospero://', - 'rtsp://', - 'service:', - 'sip:', - 'sips:', - 'sms:', - 'snmp://', - 'soap.beep://', - 'soap.beeps://', - 'tag:', - 'tel:', - 'telnet://', - 'tftp://', - 'thismessage:/', - 'tip://', - 'tv:', - 'urn://', - 'vemmi://', - 'xmlrpc.beep://', - 'xmlrpc.beeps://', - 'xmpp:', - 'z39.50r://', - 'z39.50s://', - 'about:', - 'apt:', - 'cvs://', - 'daap://', - 'ed2k://', - 'feed:', - 'fish://', - 'git://', - 'iax2:', - 'irc://', - 'ircs://', - 'ldaps://', - 'magnet:', - 'mms://', - 'rsync://', - 'ssh://', - 'svn://', - 'sftp://', - 'smb://', - 'webcal://', - 'aesgcm://', -} -

# This is an excerpt of Media Types from # https://www.iana.org/assignments/media-types/media-types.xhtml

===================================== gajim/common/helpers.py ===================================== @@ -87,6 +87,7 @@

from gajim.common import app from gajim.common import configpaths +from gajim.common import iana from gajim.common.i18n import Q_ from gajim.common.i18n import _ from gajim.common.i18n import ngettext @@ -922,6 +923,12 @@ def func_wrapper(self, *args, **kwargs): return func_wrapper

+def is_registered_uri_scheme(scheme: str) -> bool: + if scheme in iana.URI_SCHEMES: + return True + return scheme in app.settings.get('additional_uri_schemes').split() + + def parse_xmpp_uri_query(pct_iquerycomp: str) -> tuple[str, dict[str, str]]: ''' Parses 'mess%61ge;b%6Fdy=Hello%20%F0%9F%8C%90%EF%B8%8F' into @@ -959,6 +966,9 @@ def parse_uri(uri: str) -> URI:

scheme = urlparts.scheme # urlparse is expected to return it in lower case

+ if not is_registered_uri_scheme(scheme): + return URI(URIType.INVALID, uri, data={'error': 'Unregistered scheme'}) + if scheme in ('https', 'http'): if not urlparts.netloc: err = f'No host or empty host in an {scheme} URI'

===================================== gajim/common/iana.py ===================================== @@ -0,0 +1,368 @@ +# Generated by get_iana_data.py @ 2022-11-30T19:55:30.680829 + +URI_SCHEMES = { + 'aaa', + 'aaas', + 'about', + 'acap', + 'acct', + 'acd', + 'acr', + 'adiumxtra', + 'adt', + 'afp', + 'afs', + 'aim', + 'amss', + 'android', + 'appdata', + 'apt', + 'ar', + 'ark', + 'attachment', + 'aw', + 'barion', + 'bb', + 'beshare', + 'bitcoin', + 'bitcoincash', + 'blob', + 'bolo', + 'browserext', + 'cabal', + 'calculator', + 'callto', + 'cap', + 'cast', + 'casts', + 'chrome', + 'chrome-extension', + 'cid', + 'coap', + 'coap+tcp', + 'coap+ws', + 'coaps', + 'coaps+tcp', + 'coaps+ws', + 'com-eventbrite-attendee', + 'content', + 'content-type', + 'crid', + 'cstr', + 'cvs', + 'dab', + 'dat', + 'data', + 'dav', + 'diaspora', + 'dict', + 'did', + 'dis', + 'dlna-playcontainer', + 'dlna-playsingle', + 'dns', + 'dntp', + 'doi', + 'dpp', + 'drm', + 'drop', + 'dtmi', + 'dtn', + 'dvb', + 'dvx', + 'dweb', + 'ed2k', + 'eid', + 'elsi', + 'embedded', + 'ens', + 'ethereum', + 'example', + 'facetime', + 'fax', + 'feed', + 'feedready', + 'fido', + 'file', + 'filesystem', + 'finger', + 'first-run-pen-experience', + 'fish', + 'fm', + 'ftp', + 'fuchsia-pkg', + 'geo', + 'gg', + 'git', + 'gitoid', + 'gizmoproject', + 'go', + 'gopher', + 'graph', + 'grd', + 'gtalk', + 'h323', + 'ham', + 'hcap', + 'hcp', + 'http', + 'https', + 'hxxp', + 'hxxps', + 'hydrazone', + 'hyper', + 'iax', + 'icap', + 'icon', + 'im', + 'imap', + 'info', + 'iotdisco', + 'ipfs', + 'ipn', + 'ipns', + 'ipp', + 'ipps', + 'irc', + 'irc6', + 'ircs', + 'iris', + 'iris.beep', + 'iris.lwz', + 'iris.xpc', + 'iris.xpcs', + 'isostore', + 'itms', + 'jabber', + 'jar', + 'jms', + 'keyparc', + 'lastfm', + 'lbry', + 'ldap', + 'ldaps', + 'leaptofrogans', + 'lorawan', + 'lpa', + 'lvlt', + 'magnet', + 'mailserver', + 'mailto', + 'maps', + 'market', + 'matrix', + 'message', + 'microsoft.windows.camera', + 'microsoft.windows.camera.multipicker', + 'microsoft.windows.camera.picker', + 'mid', + 'mms', + 'modem', + 'mongodb', + 'moz', + 'ms-access', + 'ms-appinstaller', + 'ms-browser-extension', + 'ms-calculator', + 'ms-drive-to', + 'ms-enrollment', + 'ms-excel', + 'ms-eyecontrolspeech', + 'ms-gamebarservices', + 'ms-gamingoverlay', + 'ms-getoffice', + 'ms-help', + 'ms-infopath', + 'ms-inputapp', + 'ms-lockscreencomponent-config', + 'ms-media-stream-id', + 'ms-meetnow', + 'ms-mixedrealitycapture', + 'ms-mobileplans', + 'ms-newsandinterests', + 'ms-officeapp', + 'ms-people', + 'ms-project', + 'ms-powerpoint', + 'ms-publisher', + 'ms-restoretabcompanion', + 'ms-screenclip', + 'ms-screensketch', + 'ms-search', + 'ms-search-repair', + 'ms-secondary-screen-controller', + 'ms-secondary-screen-setup', + 'ms-settings', + 'ms-settings-airplanemode', + 'ms-settings-bluetooth', + 'ms-settings-camera', + 'ms-settings-cellular', + 'ms-settings-cloudstorage', + 'ms-settings-connectabledevices', + 'ms-settings-displays-topology', + 'ms-settings-emailandaccounts', + 'ms-settings-language', + 'ms-settings-location', + 'ms-settings-lock', + 'ms-settings-nfctransactions', + 'ms-settings-notifications', + 'ms-settings-power', + 'ms-settings-privacy', + 'ms-settings-proximity', + 'ms-settings-screenrotation', + 'ms-settings-wifi', + 'ms-settings-workplace', + 'ms-spd', + 'ms-stickers', + 'ms-sttoverlay', + 'ms-transit-to', + 'ms-useractivityset', + 'ms-virtualtouchpad', + 'ms-visio', + 'ms-walk-to', + 'ms-whiteboard', + 'ms-whiteboard-cmd', + 'ms-word', + 'msnim', + 'msrp', + 'msrps', + 'mss', + 'mt', + 'mtqp', + 'mumble', + 'mupdate', + 'mvn', + 'news', + 'nfs', + 'ni', + 'nih', + 'nntp', + 'notes', + 'num', + 'ocf', + 'oid', + 'onenote', + 'onenote-cmd', + 'opaquelocktoken', + 'openpgp4fpr', + 'otpauth', + 'p1', + 'pack', + 'palm', + 'paparazzi', + 'payment', + 'payto', + 'pkcs11', + 'platform', + 'pop', + 'pres', + 'prospero', + 'proxy', + 'pwid', + 'psyc', + 'pttp', + 'qb', + 'query', + 'quic-transport', + 'redis', + 'rediss', + 'reload', + 'res', + 'resource', + 'rmi', + 'rsync', + 'rtmfp', + 'rtmp', + 'rtsp', + 'rtsps', + 'rtspu', + 'sarif', + 'secondlife', + 'secret-token', + 'service', + 'session', + 'sftp', + 'sgn', + 'shc', + 'shttp', + 'sieve', + 'simpleledger', + 'simplex', + 'sip', + 'sips', + 'skype', + 'smb', + 'smp', + 'sms', + 'smtp', + 'snews', + 'snmp', + 'soap.beep', + 'soap.beeps', + 'soldat', + 'spiffe', + 'spotify', + 'ssb', + 'ssh', + 'steam', + 'stun', + 'stuns', + 'submit', + 'svn', + 'swh', + 'swid', + 'swidpath', + 'tag', + 'taler', + 'teamspeak', + 'tel', + 'teliaeid', + 'telnet', + 'tftp', + 'things', + 'thismessage', + 'tip', + 'tn3270', + 'tool', + 'turn', + 'turns', + 'tv', + 'udp', + 'unreal', + 'upt', + 'urn', + 'ut2004', + 'uuid-in-package', + 'v-event', + 'vemmi', + 'ventrilo', + 'ves', + 'videotex', + 'vnc', + 'view-source', + 'vscode', + 'vscode-insiders', + 'vsls', + 'w3', + 'wais', + 'web3', + 'wcr', + 'webcal', + 'wifi', + 'wpid', + 'ws', + 'wss', + 'wtai', + 'wyciwyg', + 'xcon', + 'xcon-userid', + 'xfire', + 'xmlrpc.beep', + 'xmlrpc.beeps', + 'xmpp', + 'xri', + 'ymsgr', + 'z39.50', + 'z39.50r', + 'z39.50s', +}

===================================== gajim/common/setting_values.py ===================================== @@ -36,6 +36,7 @@ class _ACCOUNT_DEFAULT:

BoolSettings = Literal[ + 'additional_uri_schemes', 'allow_open_file_uris', 'always_english_wikipedia', 'always_english_wiktionary', @@ -164,6 +165,7 @@ class _ACCOUNT_DEFAULT: AllSettingsT = Union[str, int, bool, list[str]]

APP_SETTINGS = { + 'additional_uri_schemes': '', 'allow_open_file_uris': False, 'always_english_wikipedia': False, 'always_english_wiktionary': True, @@ -628,6 +630,9 @@ class WorkspaceSettings(TypedDict):

ADVANCED_SETTINGS = { 'app': { + 'additional_uri_schemes': _( + 'Clickable schemes in addition to the hard-coded list of ' + 'IANA-registered ones. Space-separated, lower-case, no colons.'), 'allow_open_file_uris': _('Allows opening file:/ URIs when clicked.'), 'always_english_wikipedia': '', 'always_english_wiktionary': '',

===================================== gajim/common/styling.py ===================================== @@ -50,7 +50,8 @@ BLOCK_NESTED_RX = re.compile(PRE_NESTED_RX + '|' + QUOTE_RX, re.S | re.M) UNQUOTE_RX = re.compile(r'^> |^>', re.M)

-URI_OR_JID_RX = re.compile(fr'(?P<uri>{regex.IRI})|(?P<jid>{regex.XMPP.jid})') +URI_OR_JID_RX = re.compile( + fr'(?P<uri>(?<![\w+.-]){regex.IRI})|(?P<jid>{regex.XMPP.jid})')

EMOJI_RX = emoji_data.get_regex() EMOJI_RX = re.compile(EMOJI_RX)

===================================== scripts/get_iana_data.py ===================================== @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +import argparse +from datetime import datetime +from urllib.request import urlopen +import csv +import io +import logging +from pathlib import Path +import re +import sys + + +SCHEMES_URL = 'https://www.iana.org/assignments/uri-schemes/uri-schemes-1.csv' + +logging.basicConfig(level='INFO', format='%(levelname)s: %(message)s') + + +def download_file(url: str) -> str: + logging.info('Download: %s', url) + with urlopen(url) as f: + content = f.read() + return content.decode() + + +def parse_uri_schemes(content: str) -> list[str]: + logging.info('Parse uri schemes') + schemes: list[str] = [] + + reader = csv.reader(io.StringIO(content), delimiter=',') + next(reader) # Skip header row + for line in reader: + scheme = line[0].lower().removesuffix(' (obsolete)') + if not re.fullmatch('[a-z0-9+.-]+', scheme): + logging.warning('unexpected scheme field contents: %s', scheme) + continue + + schemes.append(scheme) + + return schemes + + +def generate_output(schemes: list[str], + out_path: Path): + + logging.info('Generate output') + current_date = datetime.utcnow().isoformat() + content = f'# Generated by get_iana_data.py @ {current_date}\n' + + content += '\n' + + content += 'URI_SCHEMES = {\n' + for scheme in schemes: + content += f" '{scheme}',\n" + content += '}\n' + + outpath.write_text(content) + logging.info('Wrote file to %s', outpath) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate IANA data') + parser.add_argument('out', help='Path to output file') + args = parser.parse_args() + + outpath = Path(args.out) + if outpath.is_dir(): + sys.exit('Output path is a directory') + + scheme_content = download_file(SCHEMES_URL) + schemes = parse_uri_schemes(scheme_content) + output = generate_output(schemes, + outpath) + logging.info('Finished !')

===================================== test/no_gui/test_styling.py ===================================== @@ -1,4 +1,5 @@ import unittest +from unittest.mock import MagicMock

from gajim import gui gui.init('gtk') @@ -17,6 +18,10 @@ from gajim.common.text_helpers import jid_to_iri

+app.settings = MagicMock() +app.settings.get.return_value = 'a a- a. scheme' # additional_uri_schemes + + STYLING = { 'pre cannot have children': { 'input': '_no pre `with *children*`_', @@ -384,6 +389,14 @@ ' ', '.', ':', + '://', + 'Böblingen:🌥', + 'Bo\u0308blingen:🌥', # finds blingen:🌥 but rejects an unregistered scheme + 'path/to/file_na.me:123', + '_sche.me:body_', + '_sche+me:body_', + '.scheme:body', + '+scheme:body',

# These are from https://mathiasbynens.be/demo/url-regex '//', @@ -391,10 +404,6 @@ '///a', '///', 'foo.com', - 'http://foo.bar?q=Spaces should be encoded', - 'http:// shouldfail.com', - ':// should fail', - 'http://foo.bar/foo(bar)baz quux', ]

@@ -427,6 +436,8 @@ 'file:a', 'file:a/', 'file:a/b', + + 'mailtomailto:foo@bar.com.uk', ]

@@ -484,7 +495,6 @@

# These are from https://rfc-editor.org/rfc/rfc7622#section-3.5 '"juliet"@example.com', - 'foo bar@example.com', # search is expected to find 'bar@example.com' '@example.com', 'henryⅣ@example.com', # localpart has a compatibility-decomposable cp '♚@example.com', # localpart has a symbol cp @@ -495,13 +505,14 @@ URIS_WITH_TEXT = [ ('write to my email mailto:foo@bar.com.uk (but not to mailto:bar@foo.com)', ['mailto:foo@bar.com.uk', 'mailto:bar@foo.com']), - ('write to my email mailtomailto:foo@bar.com.uk (but not to mailto:bar@foo.com)', - ['mailtomailto:foo@bar.com.uk', 'mailto:bar@foo.com']), ('see this http://userid@example.com/ link', ['http://userid@example.com/']), ('see this http://userid@example.com/, and ..', ['http://userid@example.com/']), ('http://userid@example.com/', ['http://userid@example.com/']), ('"http://userid@example.com/"', ['http://userid@example.com/']), ('regexes are useless (see https://en.wikipedia.org/wiki/Recursion_(computer_science)), but comfy', ['https://en.wikipedia.org/wiki/Recursion_(computer_science)']), + ('(scheme:body)', ['scheme:body']), + ('/scheme:body', ['scheme:body']), + ('!scheme:body', ['scheme:body']), ]

@@ -519,41 +530,32 @@ def test_uris(self): for uri in URIS: text = self.wrap(uri) hlinks = process_uris(text) - self.assertEqual(len(hlinks), 1, text) - self.assertEqual(hlinks[0].uri, uri, text) + self.assertEqual([l.uri for l in hlinks], [uri], text)

def test_invalid_uris(self): for foo in NONURIS + UNACCEPTABLE_URIS: text = self.wrap(foo) hlinks = process_uris(text) - if len(hlinks) == 0: - continue - self.assertEqual(len(hlinks), 1, text) - self.assertNotEqual(hlinks[0].text, foo, text) + self.assertEqual([l.text for l in hlinks], [], text)

def test_jids(self): for jid in JIDS: text = self.wrap(jid) + uri = jid_to_iri(jid) hlinks = process_uris(text) - self.assertEqual(len(hlinks), 1, text) - self.assertEqual(hlinks[0].text, jid, text) - self.assertEqual(hlinks[0].uri, jid_to_iri(jid), text) + self.assertEqual([(l.text, l.uri) for l in hlinks], + [(jid, uri)], text)

def test_nonjids(self): for foo in NONJIDS: text = self.wrap(foo) hlinks = process_uris(text) - if len(hlinks) == 0: - continue - self.assertEqual(len(hlinks), 1, text) - self.assertNotEqual(hlinks[0].text, foo, text) + self.assertEqual([l.text for l in hlinks], [], text)

def test_uris_with_text(self): - for text, result in URIS_WITH_TEXT: + for text, results in URIS_WITH_TEXT: hlinks = process_uris(text) - self.assertEqual(len(hlinks), len(result), text) - for i, res in enumerate(result): - self.assertEqual(hlinks[i].text, res, text) + self.assertEqual([l.text for l in hlinks], results, text)

if __name__ == "__main__":

View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/2f8986b3d3b4f83ed942b8f108b62bf9...