GitLab

at 2022-11-30T21:05:07+00:00

imprv: Styling: Introduce URI scheme whitelisting

Fixes #11266

Co-authored-by: Philipp Hörist <philipp@hoerist.com>

fix: Styling: Linkify URIs only if at the start of a word

refactor: Tests: Improve failure reports in test_styling

 

 ## Build

 * [ ] Merge translations from Weblate

+* [ ] Update IANA data with `./scripts/get_iana_data.py gajim/common/iana.py`

 * [ ] Run `./scripts/bump_version.py x.x.x`

 * [ ] Push release tag `x.x.x`

 

 GAJIM_WIKI_URI = 'https://dev.gajim.org/gajim/gajim/wikis'

 GAJIM_SUPPORT_JID = 'gajim@conference.gajim.org'

 

-URI_SCHEMES = {

-    'aaa://',

-    'aaas://',

-    'acap://',

-    'cap://',

-    'cid:',

-    'crid://',

-    'data:',

-    'dav:',

-    'dict://',

-    'dns:',

-    'fax:',

-    'file:/',

-    'ftp://',

-    'geo:',

-    'go:',

-    'gopher://',

-    'h323:',

-    'http://',

-    'https://',

-    'iax:',

-    'icap://',

-    'im:',

-    'imap://',

-    'info:',

-    'ipp://',

-    'iris:',

-    'iris.beep:',

-    'iris.xpc:',

-    'iris.xpcs:',

-    'iris.lwz:',

-    'ldap://',

-    'mailto:',

-    'mid:',

-    'modem:',

-    'msrp://',

-    'msrps://',

-    'mtqp://',

-    'mupdate://',

-    'news:',

-    'nfs://',

-    'nntp://',

-    'opaquelocktoken:',

-    'pop://',

-    'pres:',

-    'prospero://',

-    'rtsp://',

-    'service:',

-    'sip:',

-    'sips:',

-    'sms:',

-    'snmp://',

-    'soap.beep://',

-    'soap.beeps://',

-    'tag:',

-    'tel:',

-    'telnet://',

-    'tftp://',

-    'thismessage:/',

-    'tip://',

-    'tv:',

-    'urn://',

-    'vemmi://',

-    'xmlrpc.beep://',

-    'xmlrpc.beeps://',

-    'xmpp:',

-    'z39.50r://',

-    'z39.50s://',

-    'about:',

-    'apt:',

-    'cvs://',

-    'daap://',

-    'ed2k://',

-    'feed:',

-    'fish://',

-    'git://',

-    'iax2:',

-    'irc://',

-    'ircs://',

-    'ldaps://',

-    'magnet:',

-    'mms://',

-    'rsync://',

-    'ssh://',

-    'svn://',

-    'sftp://',

-    'smb://',

-    'webcal://',

-    'aesgcm://',

-}

-

 

 # This is an excerpt of Media Types from

 # https://www.iana.org/assignments/media-types/media-types.xhtml

 

 from gajim.common import app

 from gajim.common import configpaths

+from gajim.common import iana

 from gajim.common.i18n import Q_

 from gajim.common.i18n import _

 from gajim.common.i18n import ngettext

     return func_wrapper

 

 

+def is_registered_uri_scheme(scheme: str) -> bool:

+    if scheme in iana.URI_SCHEMES:

+        return True

+    return scheme in app.settings.get('additional_uri_schemes').split()

+

+

 def parse_xmpp_uri_query(pct_iquerycomp: str) -> tuple[str, dict[str, str]]:

     '''

     Parses 'mess%61ge;b%6Fdy=Hello%20%F0%9F%8C%90%EF%B8%8F' into

 

     scheme = urlparts.scheme  # urlparse is expected to return it in lower case

 

+    if not is_registered_uri_scheme(scheme):

+        return URI(URIType.INVALID, uri, data={'error': 'Unregistered scheme'})

+

     if scheme in ('https', 'http'):

         if not urlparts.netloc:

             err = f'No host or empty host in an {scheme} URI'

+# Generated by get_iana_data.py @ 2022-11-30T19:55:30.680829

+

+URI_SCHEMES = {

+    'aaa',

+    'aaas',

+    'about',

+    'acap',

+    'acct',

+    'acd',

+    'acr',

+    'adiumxtra',

+    'adt',

+    'afp',

+    'afs',

+    'aim',

+    'amss',

+    'android',

+    'appdata',

+    'apt',

+    'ar',

+    'ark',

+    'attachment',

+    'aw',

+    'barion',

+    'bb',

+    'beshare',

+    'bitcoin',

+    'bitcoincash',

+    'blob',

+    'bolo',

+    'browserext',

+    'cabal',

+    'calculator',

+    'callto',

+    'cap',

+    'cast',

+    'casts',

+    'chrome',

+    'chrome-extension',

+    'cid',

+    'coap',

+    'coap+tcp',

+    'coap+ws',

+    'coaps',

+    'coaps+tcp',

+    'coaps+ws',

+    'com-eventbrite-attendee',

+    'content',

+    'content-type',

+    'crid',

+    'cstr',

+    'cvs',

+    'dab',

+    'dat',

+    'data',

+    'dav',

+    'diaspora',

+    'dict',

+    'did',

+    'dis',

+    'dlna-playcontainer',

+    'dlna-playsingle',

+    'dns',

+    'dntp',

+    'doi',

+    'dpp',

+    'drm',

+    'drop',

+    'dtmi',

+    'dtn',

+    'dvb',

+    'dvx',

+    'dweb',

+    'ed2k',

+    'eid',

+    'elsi',

+    'embedded',

+    'ens',

+    'ethereum',

+    'example',

+    'facetime',

+    'fax',

+    'feed',

+    'feedready',

+    'fido',

+    'file',

+    'filesystem',

+    'finger',

+    'first-run-pen-experience',

+    'fish',

+    'fm',

+    'ftp',

+    'fuchsia-pkg',

+    'geo',

+    'gg',

+    'git',

+    'gitoid',

+    'gizmoproject',

+    'go',

+    'gopher',

+    'graph',

+    'grd',

+    'gtalk',

+    'h323',

+    'ham',

+    'hcap',

+    'hcp',

+    'http',

+    'https',

+    'hxxp',

+    'hxxps',

+    'hydrazone',

+    'hyper',

+    'iax',

+    'icap',

+    'icon',

+    'im',

+    'imap',

+    'info',

+    'iotdisco',

+    'ipfs',

+    'ipn',

+    'ipns',

+    'ipp',

+    'ipps',

+    'irc',

+    'irc6',

+    'ircs',

+    'iris',

+    'iris.beep',

+    'iris.lwz',

+    'iris.xpc',

+    'iris.xpcs',

+    'isostore',

+    'itms',

+    'jabber',

+    'jar',

+    'jms',

+    'keyparc',

+    'lastfm',

+    'lbry',

+    'ldap',

+    'ldaps',

+    'leaptofrogans',

+    'lorawan',

+    'lpa',

+    'lvlt',

+    'magnet',

+    'mailserver',

+    'mailto',

+    'maps',

+    'market',

+    'matrix',

+    'message',

+    'microsoft.windows.camera',

+    'microsoft.windows.camera.multipicker',

+    'microsoft.windows.camera.picker',

+    'mid',

+    'mms',

+    'modem',

+    'mongodb',

+    'moz',

+    'ms-access',

+    'ms-appinstaller',

+    'ms-browser-extension',

+    'ms-calculator',

+    'ms-drive-to',

+    'ms-enrollment',

+    'ms-excel',

+    'ms-eyecontrolspeech',

+    'ms-gamebarservices',

+    'ms-gamingoverlay',

+    'ms-getoffice',

+    'ms-help',

+    'ms-infopath',

+    'ms-inputapp',

+    'ms-lockscreencomponent-config',

+    'ms-media-stream-id',

+    'ms-meetnow',

+    'ms-mixedrealitycapture',

+    'ms-mobileplans',

+    'ms-newsandinterests',

+    'ms-officeapp',

+    'ms-people',

+    'ms-project',

+    'ms-powerpoint',

+    'ms-publisher',

+    'ms-restoretabcompanion',

+    'ms-screenclip',

+    'ms-screensketch',

+    'ms-search',

+    'ms-search-repair',

+    'ms-secondary-screen-controller',

+    'ms-secondary-screen-setup',

+    'ms-settings',

+    'ms-settings-airplanemode',

+    'ms-settings-bluetooth',

+    'ms-settings-camera',

+    'ms-settings-cellular',

+    'ms-settings-cloudstorage',

+    'ms-settings-connectabledevices',

+    'ms-settings-displays-topology',

+    'ms-settings-emailandaccounts',

+    'ms-settings-language',

+    'ms-settings-location',

+    'ms-settings-lock',

+    'ms-settings-nfctransactions',

+    'ms-settings-notifications',

+    'ms-settings-power',

+    'ms-settings-privacy',

+    'ms-settings-proximity',

+    'ms-settings-screenrotation',

+    'ms-settings-wifi',

+    'ms-settings-workplace',

+    'ms-spd',

+    'ms-stickers',

+    'ms-sttoverlay',

+    'ms-transit-to',

+    'ms-useractivityset',

+    'ms-virtualtouchpad',

+    'ms-visio',

+    'ms-walk-to',

+    'ms-whiteboard',

+    'ms-whiteboard-cmd',

+    'ms-word',

+    'msnim',

+    'msrp',

+    'msrps',

+    'mss',

+    'mt',

+    'mtqp',

+    'mumble',

+    'mupdate',

+    'mvn',

+    'news',

+    'nfs',

+    'ni',

+    'nih',

+    'nntp',

+    'notes',

+    'num',

+    'ocf',

+    'oid',

+    'onenote',

+    'onenote-cmd',

+    'opaquelocktoken',

+    'openpgp4fpr',

+    'otpauth',

+    'p1',

+    'pack',

+    'palm',

+    'paparazzi',

+    'payment',

+    'payto',

+    'pkcs11',

+    'platform',

+    'pop',

+    'pres',

+    'prospero',

+    'proxy',

+    'pwid',

+    'psyc',

+    'pttp',

+    'qb',

+    'query',

+    'quic-transport',

+    'redis',

+    'rediss',

+    'reload',

+    'res',

+    'resource',

+    'rmi',

+    'rsync',

+    'rtmfp',

+    'rtmp',

+    'rtsp',

+    'rtsps',

+    'rtspu',

+    'sarif',

+    'secondlife',

+    'secret-token',

+    'service',

+    'session',

+    'sftp',

+    'sgn',

+    'shc',

+    'shttp',

+    'sieve',

+    'simpleledger',

+    'simplex',

+    'sip',

+    'sips',

+    'skype',

+    'smb',

+    'smp',

+    'sms',

+    'smtp',

+    'snews',

+    'snmp',

+    'soap.beep',

+    'soap.beeps',

+    'soldat',

+    'spiffe',

+    'spotify',

+    'ssb',

+    'ssh',

+    'steam',

+    'stun',

+    'stuns',

+    'submit',

+    'svn',

+    'swh',

+    'swid',

+    'swidpath',

+    'tag',

+    'taler',

+    'teamspeak',

+    'tel',

+    'teliaeid',

+    'telnet',

+    'tftp',

+    'things',

+    'thismessage',

+    'tip',

+    'tn3270',

+    'tool',

+    'turn',

+    'turns',

+    'tv',

+    'udp',

+    'unreal',

+    'upt',

+    'urn',

+    'ut2004',

+    'uuid-in-package',

+    'v-event',

+    'vemmi',

+    'ventrilo',

+    'ves',

+    'videotex',

+    'vnc',

+    'view-source',

+    'vscode',

+    'vscode-insiders',

+    'vsls',

+    'w3',

+    'wais',

+    'web3',

+    'wcr',

+    'webcal',

+    'wifi',

+    'wpid',

+    'ws',

+    'wss',

+    'wtai',

+    'wyciwyg',

+    'xcon',

+    'xcon-userid',

+    'xfire',

+    'xmlrpc.beep',

+    'xmlrpc.beeps',

+    'xmpp',

+    'xri',

+    'ymsgr',

+    'z39.50',

+    'z39.50r',

+    'z39.50s',

+}
 

 

 BoolSettings = Literal[

+    'additional_uri_schemes',

     'allow_open_file_uris',

     'always_english_wikipedia',

     'always_english_wiktionary',

 AllSettingsT = Union[str, int, bool, list[str]]

 

 APP_SETTINGS = {

+    'additional_uri_schemes': '',

     'allow_open_file_uris': False,

     'always_english_wikipedia': False,

     'always_english_wiktionary': True,

 

 ADVANCED_SETTINGS = {

     'app': {

+        'additional_uri_schemes': _(

+            'Clickable schemes in addition to the hard-coded list of '

+            'IANA-registered ones. Space-separated, lower-case, no colons.'),

         'allow_open_file_uris': _('Allows opening file:/ URIs when clicked.'),

         'always_english_wikipedia': '',

         'always_english_wiktionary': '',

 BLOCK_NESTED_RX = re.compile(PRE_NESTED_RX + '|' + QUOTE_RX, re.S | re.M)

 UNQUOTE_RX = re.compile(r'^> |^>', re.M)

 

-URI_OR_JID_RX = re.compile(fr'(?P<uri>{regex.IRI})|(?P<jid>{regex.XMPP.jid})')

+URI_OR_JID_RX = re.compile(

+    fr'(?P<uri>(?<![\w+.-]){regex.IRI})|(?P<jid>{regex.XMPP.jid})')

 

 EMOJI_RX = emoji_data.get_regex()

 EMOJI_RX = re.compile(EMOJI_RX)

+#!/usr/bin/env python3

+

+import argparse

+from datetime import datetime

+from urllib.request import urlopen

+import csv

+import io

+import logging

+from pathlib import Path

+import re

+import sys

+

+

+SCHEMES_URL = 'https://www.iana.org/assignments/uri-schemes/uri-schemes-1.csv'

+

+logging.basicConfig(level='INFO', format='%(levelname)s: %(message)s')

+

+

+def download_file(url: str) -> str:

+    logging.info('Download: %s', url)

+    with urlopen(url) as f:

+        content = f.read()

+    return content.decode()

+

+

+def parse_uri_schemes(content: str) -> list[str]:

+    logging.info('Parse uri schemes')

+    schemes: list[str] = []

+

+    reader = csv.reader(io.StringIO(content), delimiter=',')

+    next(reader)  # Skip header row

+    for line in reader:

+        scheme = line[0].lower().removesuffix(' (obsolete)')

+        if not re.fullmatch('[a-z0-9+.-]+', scheme):

+            logging.warning('unexpected scheme field contents: %s', scheme)

+            continue

+

+        schemes.append(scheme)

+

+    return schemes

+

+

+def generate_output(schemes: list[str],

+                    out_path: Path):

+

+    logging.info('Generate output')

+    current_date = datetime.utcnow().isoformat()

+    content = f'# Generated by get_iana_data.py @ {current_date}\n'

+

+    content += '\n'

+

+    content += 'URI_SCHEMES = {\n'

+    for scheme in schemes:

+        content += f"    '{scheme}',\n"

+    content += '}\n'

+

+    outpath.write_text(content)

+    logging.info('Wrote file to %s', outpath)

+

+

+if __name__ == '__main__':

+    parser = argparse.ArgumentParser(description='Generate IANA data')

+    parser.add_argument('out', help='Path to output file')

+    args = parser.parse_args()

+

+    outpath = Path(args.out)

+    if outpath.is_dir():

+        sys.exit('Output path is a directory')

+

+    scheme_content = download_file(SCHEMES_URL)

+    schemes = parse_uri_schemes(scheme_content)

+    output = generate_output(schemes,

+                             outpath)

+    logging.info('Finished !')
 import unittest

+from unittest.mock import MagicMock

 

 from gajim import gui

 gui.init('gtk')

 from gajim.common.text_helpers import jid_to_iri

 

 

+app.settings = MagicMock()

+app.settings.get.return_value = 'a a- a. scheme'  # additional_uri_schemes

+

+

 STYLING = {

     'pre cannot have children':  {

         'input': '_no pre `with *children*`_',

     ' ',

     '.',

     ':',

+    '://',

+    'Böblingen:🌥',

+    'Bo\u0308blingen:🌥',  # finds blingen:🌥 but rejects an unregistered scheme

+    'path/to/file_na.me:123',

+    '_sche.me:body_',

+    '_sche+me:body_',

+    '.scheme:body',

+    '+scheme:body',

 

     # These are from https://mathiasbynens.be/demo/url-regex

     '//',

     '///a',

     '///',

     'foo.com',

-    'http://foo.bar?q=Spaces should be encoded',

-    'http:// shouldfail.com',

-    ':// should fail',

-    'http://foo.bar/foo(bar)baz quux',

 ]

 

 

     'file:a',

     'file:a/',

     'file:a/b',

+

+    'mailtomailto:foo@bar.com.uk',

 ]

 

 

 

     # These are from https://rfc-editor.org/rfc/rfc7622#section-3.5

     '"juliet"@example.com',

-    'foo bar@example.com',  # search is expected to find 'bar@example.com'

     '@example.com',

     'henryⅣ@example.com',  # localpart has a compatibility-decomposable cp

     '♚@example.com',        # localpart has a symbol cp

 URIS_WITH_TEXT = [

     ('write to my email mailto:foo@bar.com.uk (but not to mailto:bar@foo.com)',

      ['mailto:foo@bar.com.uk', 'mailto:bar@foo.com']),

-    ('write to my email mailtomailto:foo@bar.com.uk (but not to mailto:bar@foo.com)',

-     ['mailtomailto:foo@bar.com.uk', 'mailto:bar@foo.com']),

     ('see this http://userid@example.com/ link', ['http://userid@example.com/']),

     ('see this http://userid@example.com/, and ..', ['http://userid@example.com/']),

     ('<http://userid@example.com/>', ['http://userid@example.com/']),

     ('"http://userid@example.com/"', ['http://userid@example.com/']),

     ('regexes are useless (see https://en.wikipedia.org/wiki/Recursion_(computer_science)), but comfy', ['https://en.wikipedia.org/wiki/Recursion_(computer_science)']),

+    ('(scheme:body)', ['scheme:body']),

+    ('/scheme:body', ['scheme:body']),

+    ('!scheme:body', ['scheme:body']),

 ]

 

 

         for uri in URIS:

             text = self.wrap(uri)

             hlinks = process_uris(text)

-            self.assertEqual(len(hlinks), 1, text)

-            self.assertEqual(hlinks[0].uri, uri, text)

+            self.assertEqual([l.uri for l in hlinks], [uri], text)

 

     def test_invalid_uris(self):

         for foo in NONURIS + UNACCEPTABLE_URIS:

             text = self.wrap(foo)

             hlinks = process_uris(text)

-            if len(hlinks) == 0:

-                continue

-            self.assertEqual(len(hlinks), 1, text)

-            self.assertNotEqual(hlinks[0].text, foo, text)

+            self.assertEqual([l.text for l in hlinks], [], text)

 

     def test_jids(self):

         for jid in JIDS:

             text = self.wrap(jid)

+            uri = jid_to_iri(jid)

             hlinks = process_uris(text)

-            self.assertEqual(len(hlinks), 1, text)

-            self.assertEqual(hlinks[0].text, jid, text)

-            self.assertEqual(hlinks[0].uri, jid_to_iri(jid), text)

+            self.assertEqual([(l.text, l.uri) for l in hlinks],

+                             [(jid, uri)], text)

 

     def test_nonjids(self):

         for foo in NONJIDS:

             text = self.wrap(foo)

             hlinks = process_uris(text)

-            if len(hlinks) == 0:

-                continue

-            self.assertEqual(len(hlinks), 1, text)

-            self.assertNotEqual(hlinks[0].text, foo, text)

+            self.assertEqual([l.text for l in hlinks], [], text)

 

     def test_uris_with_text(self):

-        for text, result in URIS_WITH_TEXT:

+        for text, results in URIS_WITH_TEXT:

             hlinks = process_uris(text)

-            self.assertEqual(len(hlinks), len(result), text)

-            for i, res in enumerate(result):

-                self.assertEqual(hlinks[i].text, res, text)

+            self.assertEqual([l.text for l in hlinks], results, text)

 

 

 if __name__ == "__main__":

Philipp Hörist pushed to branch master at gajim / gajim

Commits:

8 changed files:

Changes: