Philipp Hörist pushed to branch master at gajim / gajim
Commits: 1ef1e8fe by mjk at 2022-10-24T17:50:41+00:00 fix: Styling: Various link parsing issues
Also, a small feature: tooltip for JIDs that shows the xmpp URI.
Regression: words starting with 'www.' are no longer linkified.
Fixes #11218, #11144
- - - - -
8 changed files:
- gajim/common/regex.py - gajim/common/styling.py - + gajim/common/text_helpers.py - gajim/gtk/conversation/plain_widget.py - setup.cfg - test/no_gui/test_regex.py - test/no_gui/test_styling.py - + test/no_gui/test_text_helpers.py
Changes:
===================================== gajim/common/regex.py ===================================== @@ -13,6 +13,7 @@ # along with Gajim. If not, see http://www.gnu.org/licenses/.
import re +from typing import Any
INVALID_XML_CHARS_REGEX = re.compile( @@ -21,3 +22,101 @@ # at least one character in 3 parts (before @, after @, after .) STH_AT_STH_DOT_STH_REGEX = re.compile( r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+.[a-zA-Z0-9-.]+$') + + +# Universal/Internationalized Resource Identifiers and their components, as +# defined in [ABNF for IRI References and IRIs] +# https://rfc-editor.org/rfc/rfc3987#section-2.2 and +# [Representing IPv6 Zone Identifiers in Address Literals and URIs] +# https://www.rfc-editor.org/rfc/rfc6874. +ALPHA = r'[A-Za-z]' +HEXDIG = r'[0-9A-Fa-f]' +sub_delims = r"!$&'()*+,;=" +ucschar = r'\xA0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF'\ + r'\U00010000-\U0001FFFD\U00020000-\U0002FFFD\U00030000-\U0003FFFD'\ + r'\U00040000-\U0004FFFD\U00050000-\U0005FFFD\U00060000-\U0006FFFD'\ + r'\U00070000-\U0007FFFD\U00080000-\U0008FFFD\U00090000-\U0009FFFD'\ + r'\U000A0000-\U000AFFFD\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD'\ + r'\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD' +iprivate = r'\uE000-\uF8FF'\ + r'\U000F0000-\U000FFFFD\U00100000-\U0010FFFD' +unreserved = r'A-Za-z0-9-._~' +iunreserved = fr'{unreserved}{ucschar}' +pct_encoded = fr'%{HEXDIG}{HEXDIG}' +ipchar = fr'(?:[{iunreserved}{sub_delims}:@]|{pct_encoded})' +iuserinfo = fr'(?:[{iunreserved}{sub_delims}:]|{pct_encoded})*' +ireg_name = fr'(?:[{iunreserved}{sub_delims}]|{pct_encoded})*' +dec_octet = r'(?:25[0-5]'\ + r'|2[0-4][0-9]'\ + r'|1[0-9][0-9]'\ + r'|[1-9][0-9]'\ + r'|[0-9])' +IPv4address = fr'{dec_octet}(?:.{dec_octet}){{3}}' +h16 = fr'{HEXDIG}{{1,4}}' +ls32 = fr'(?:{h16}:{h16}|{IPv4address})' +IPv6address =\ + fr'(?:(?:{h16}:){{6}}{ls32}'\ + fr'|::(?:{h16}:){{5}}{ls32}'\ + fr'|(?:{h16})?::(?:{h16}:){{4}}{ls32}'\ + fr'|(?:(?:{h16}:){{,1}}{h16})?::(?:{h16}:){{3}}{ls32}'\ + fr'|(?:(?:{h16}:){{,2}}{h16})?::(?:{h16}:){{2}}{ls32}'\ + fr'|(?:(?:{h16}:){{,3}}{h16})?::{h16}:{ls32}'\ + fr'|(?:(?:{h16}:){{,4}}{h16})?::{ls32}'\ + fr'|(?:(?:{h16}:){{,5}}{h16})?::{h16}'\ + fr'|(?:(?:{h16}:){{,6}}{h16})?::)' # noqa: E126,E127,E131 +ZoneID = fr'(?:{unreserved}|{pct_encoded})+' +IPv6addrz = fr'{IPv6address}%25{ZoneID}' +IPvFuture = fr'[Vv]{HEXDIG}+.[{unreserved}{sub_delims}:]+' +IP_literal = fr'[(?:{IPv6address}|{IPv6addrz}|{IPvFuture})]' +# ihost = fr'(?P<host>{IP_literal}|{IPv4address}|{ireg_name})' +# The below is equivalent to the above for the purpose of validation, but +# better for the purpose of search: e.g., matches 'foo://1.2.3.4.5' completely. +ihost = fr'(?P<host>{IP_literal}|{ireg_name})' +port = r'[0-9]*' +iauthority = fr'(?:{iuserinfo}@)?{ihost}(?::{port})?' +isegment = fr'{ipchar}*' +isegment_nz = fr'{ipchar}+' +ipath_abempty = fr'(?:/{isegment})*' +ipath_absolute = fr'/(?:{isegment_nz}(?:/{isegment})*)?' +ipath_rootless = fr'{isegment_nz}(?:/{isegment})*' +ipath_empty = r'' +ihier_part = fr'(?://{iauthority}{ipath_abempty}'\ + fr'|{ipath_absolute}'\ + fr'|{ipath_rootless}'\ + fr'|{ipath_empty})' +iquery = fr'(?:{ipchar}|[/?{iprivate}])*' +ifragment = fr'(?:{ipchar}|[/?])*' +scheme = fr'(?P<scheme>{ALPHA}[a-zA-Z0-9+.-]*)' +IRI = fr'{scheme}:{ihier_part}(?:?{iquery})?(?:#{ifragment})?' + + +# https://rfc-editor.org/rfc/rfc7564#section-4.2 +class PRECIS: + LetterDigits = r'\w' # roughly (e.g., includes '_' as an extra) + Spaces = r'\s' # roughly? + Symbols = r'♚' # let's make that test pass! + HasCompat = r'Ⅳ' # and that one! + IdentifierClass: Any + + +class PRECIS_IdentifierClass: + _p = PRECIS + Valid = fr'{_p.LetterDigits}\x21-\x7e' + Disallowed = fr'{_p.Spaces}{_p.Symbols}{_p.HasCompat}' # there's a ton more + + +PRECIS.IdentifierClass = PRECIS_IdentifierClass + + +# (Rough approximation of) Jabber IDs, as defined in [XMPP: Address Format] +# https://rfc-editor.org/rfc/rfc7622#section-3. +class XMPP: + _id = PRECIS.IdentifierClass + _excl = '"&'/:<>@' # <#section-3.3.1> + localpart = fr'(?:(?![{_excl}{_id.Disallowed}])[{_id.Valid}])+' + # ^ doesn't take into account "contextual rules" <rfc7564#section-4.2.2> + ifqdn = fr'[{iunreserved}{sub_delims}]+' + # ^ probably correct + domainpart = fr'(?:{IP_literal}|{IPv4address}|{ifqdn})' + jid = fr'(?P<local>{localpart})@(?P<domain>{domainpart})' + # ^ notably, doesn't include resourcepart (TODO)
===================================== gajim/common/styling.py ===================================== @@ -25,6 +25,8 @@
from gi.repository import GLib
+from gajim.common import regex +from gajim.common.text_helpers import jid_to_iri from gajim.gui.emoji_data import emoji_data
PRE = '`' @@ -44,13 +46,7 @@ BLOCK_NESTED_RX = re.compile(PRE_NESTED_RX + '|' + QUOTE_RX, re.S | re.M) UNQUOTE_RX = re.compile(r'^> |^>', re.M)
-URI_RX = r'(?P<uri>([\w-]+://|www[.])[\S()<>]+?(?=[,>]?(\s|\Z)+))' -ADDRESS_RX = r'(?P<address>\b((xmpp|mailto):)?[\w-]+(.[\w-]+)*@([\w-]*?.)+[\w]+([?].*?(?=([\s),]|$)))?)' # noqa: E501 -URI_ADDRESS_RX = URI_RX + '|' + ADDRESS_RX - -URI_RX = re.compile(URI_RX) -ADDRESS_RX = re.compile(ADDRESS_RX) -URI_ADDRESS_RX = re.compile(URI_ADDRESS_RX) +URI_OR_JID_RX = re.compile(fr'(?P<uri>{regex.IRI})|(?P<jid>{regex.XMPP.jid})')
EMOJI_RX = emoji_data.get_regex() EMOJI_RX = re.compile(EMOJI_RX) @@ -68,33 +64,31 @@ class StyleObject:
@dataclass -class BaseUri(StyleObject): +class BaseHyperlink(StyleObject): name: str + uri: str start_byte: int end_byte: int
def get_markup_string(self) -> str: + href = GLib.markup_escape_text(self.uri) text = GLib.markup_escape_text(self.text) - return f'<a href="{text}">{text}</a>' + title = '' if href == text else f' title="{href}"' + return f'<a href="{href}"{title}>{text}</a>'
@dataclass -class Uri(BaseUri): +class Hyperlink(BaseHyperlink): name: str = field(default='uri', init=False)
@dataclass -class Address(BaseUri): - name: str = field(default='address', init=False) - - -@dataclass -class XMPPAddress(BaseUri): +class XMPPAddress(BaseHyperlink): name: str = field(default='xmppadr', init=False)
@dataclass -class MailAddress(BaseUri): +class MailAddress(BaseHyperlink): name: str = field(default='mailadr', init=False)
@@ -117,7 +111,7 @@ def from_match(cls, match: Match[str]) -> Block: class PlainBlock(Block): name: str = field(default='plain', init=False) spans: list[Span] = field(default_factory=list) - uris: list[BaseUri] = field(default_factory=list) + uris: list[BaseHyperlink] = field(default_factory=list) emojis: list[Emoji] = field(default_factory=list)
@classmethod @@ -224,11 +218,11 @@ def process(text: Union[str, bytes], level: int = 0) -> ParsingResult: return ParsingResult(text, blocks)
-def process_uris(text: Union[str, bytes]) -> list[BaseUri]: +def process_uris(text: Union[str, bytes]) -> list[BaseHyperlink]: if isinstance(text, bytes): text = text.decode()
- uris: list[BaseUri] = [] + uris: list[BaseHyperlink] = [] offset = 0 offset_bytes = 0 for line in text.splitlines(keepends=True): @@ -330,20 +324,36 @@ def _parse_line(line: str, offset: int, offset_bytes: int) -> list[Span]: return spans
-def _parse_uris(line: str, offset: int, offset_bytes: int) -> list[BaseUri]: - uris: list[BaseUri] = [] - - def make(match: Match[str], is_address: bool) -> BaseUri: - return _make_link(line, - match.start(), - match.end() - 1, - offset, - offset_bytes, - is_address) - - for match in URI_ADDRESS_RX.finditer(line): - uri = make(match, bool(match.group('address'))) - uris.append(uri) +def _parse_uris(line: str, + offset: int, + offset_bytes: int) -> list[BaseHyperlink]: + uris: list[BaseHyperlink] = [] + + def make(start: int, end: int, is_jid: bool) -> BaseHyperlink: + if line[end - 1] == ',': + # Trim one trailing comma + end -= 1 + if '(' in line[:start] and line[end - 1] == ')': + # Trim one trailing closing parenthesis if the match is preceded + # by an opening one somewhere on the line + end -= 1 + return _make_hyperlink(line, + start, + end - 1, + offset, + offset_bytes, + is_jid) + + for match in URI_OR_JID_RX.finditer(line): + start, end = match.span() + if match.group('jid') is not None: + uris.append(make(start, end, True)) + elif not re.fullmatch('[^:]+:', line[start:end]) and\ + (not re.fullmatch('https?', match.group('scheme'), re.IGNORECASE) or + match.group('host')): + # URIs that consist only of a scheme are thusly excluded. + # http(s) URIs without host or with empty host are excluded too. + uris.append(make(start, end, False))
return uris
@@ -407,12 +417,12 @@ def _make_span(line: str, text=text)
-def _make_link(line: str, - start: int, - end: int, - offset: int, - offset_bytes: int, - is_address: bool) -> BaseUri: +def _make_hyperlink(line: str, + start: int, + end: int, + offset: int, + offset_bytes: int, + is_jid: bool) -> BaseHyperlink:
text = line[start:end + 1]
@@ -422,22 +432,22 @@ def _make_link(line: str, start += offset end += offset + 1
- if not is_address: - cls_ = Uri - - elif text.startswith('xmpp'): + uri = text + if is_jid: cls_ = XMPPAddress - - elif text.startswith('mailto'): + uri = jid_to_iri(text) + elif re.match('xmpp:', text, re.IGNORECASE): + cls_ = XMPPAddress + elif re.match('mailto:', text, re.IGNORECASE): cls_ = MailAddress - else: - cls_ = Address + cls_ = Hyperlink
return cls_(start=start, start_byte=start_byte, end=end, end_byte=end_byte, + uri=uri, text=text)
===================================== gajim/common/text_helpers.py ===================================== @@ -0,0 +1,33 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see http://www.gnu.org/licenses/. + +from gi.repository import GLib +from gajim.common import regex + +# from RFC 3986, 3.3. Path (pchar without unreserved and pct-encoded): +_reserved_chars_allowed_in_path_segment = regex.sub_delims + ':@' + + +def escape_iri_path_segment(s: str) -> str: + return GLib.Uri.escape_string( + s, _reserved_chars_allowed_in_path_segment, True) + + +def escape_iri_path(s: str) -> str: + return GLib.Uri.escape_string( + s, _reserved_chars_allowed_in_path_segment + '/', True) + + +def jid_to_iri(jid: str) -> str: + return 'xmpp:' + escape_iri_path(jid)
===================================== gajim/gtk/conversation/plain_widget.py ===================================== @@ -31,7 +31,7 @@ from gajim.common.helpers import open_uri from gajim.common.helpers import parse_uri from gajim.common.structs import URI -from gajim.common.styling import BaseUri +from gajim.common.styling import BaseHyperlink from gajim.common.styling import PlainBlock from gajim.common.styling import process_uris
@@ -44,7 +44,7 @@
log = logging.getLogger('gajim.gui.conversaion.plain_widget')
-URI_TAGS = ['uri', 'address', 'xmppadr', 'mailadr'] +URI_TAGS = ['uri', 'xmppadr', 'mailadr'] STYLE_TAGS = ['strong', 'emphasis', 'strike', 'pre']
@@ -106,7 +106,7 @@ def _on_populate_popup(self, label: Gtk.Label, menu: Gtk.Menu) -> None: menu.prepend(action_menu_item) menu.show_all()
- def _build_link_markup(self, text: str, uris: list[BaseUri]) -> str: + def _build_link_markup(self, text: str, uris: list[BaseHyperlink]) -> str: markup_text = '' after = GLib.markup_escape_text(text.strip()) for uri in uris:
===================================== setup.cfg ===================================== @@ -9,6 +9,7 @@ exclude = typings, gajim/common/config.py, gajim/common/optparser.py, + gajim/common/regex.py, gajim/common/socks5.py, gajim/gui, max-complexity = 15
===================================== test/no_gui/test_regex.py ===================================== @@ -1,27 +1,38 @@ from typing import Any
import unittest +import re
-from gajim.common.styling import URI_RX -from gajim.common.styling import ADDRESS_RX +from gajim.common import regex
TEST_URIS = [ + 'scheme:', + 'http:///', + 'http://', 'http://google.com/', 'http://google.com', 'http://www.google.ca/search?q=xmpp', 'http://tools.ietf.org/html/draft-saintandre-rfc3920bis-05#section-12.3', 'http://en.wikipedia.org/wiki/Protocol_(computing)', 'http://en.wikipedia.org/wiki/Protocol_%28computing%29', -] - -ADDRESSES = [ 'mailto:test@example.org', 'xmpp:example-node@example.com', - # 'xmpp:example-node@example.com/some-resource', FIXME + 'xmpp:example-node@example.com/some-resource', 'xmpp:example-node@example.com?message', ]
+JIDS = [ + 'example-node@example.com', + 'me@[::1]', + 'myself@127.13.42.69', + #'myself@127.13.42.69/localhost', TODO + '#room%irc.example@biboumi.xmpp.example', + r'here\27s_a_wild_\26_\2fcr%zy\2f_address_for\3a\3cwv\3e(\22IMPS\22)@example.com', + r'CN=D\27Artagnan\20Saint-Andr\E9,O=Example\20\26\20Company,\20Inc.,DC=example,DC=com@st.example.com', + 'δοκιμή@παράδειγμα.δοκιμή', +] +
class Test(unittest.TestCase): def test_links_regexp_entire(self): @@ -34,10 +45,10 @@ def assert_matches_all(regex: Any, uri_str: str) -> None: self.assertEqual(m.span(), str_span, uri_str)
for uri in TEST_URIS: - assert_matches_all(URI_RX, uri) + assert_matches_all(re.compile(regex.IRI), uri)
- for addr in ADDRESSES: - assert_matches_all(ADDRESS_RX, addr) + for addr in JIDS: + assert_matches_all(re.compile(regex.XMPP.jid), addr)
if __name__ == '__main__':
===================================== test/no_gui/test_styling.py ===================================== @@ -8,9 +8,9 @@ from gajim.common.styling import StrongSpan from gajim.common.styling import EmphasisSpan from gajim.common.styling import StrikeSpan -from gajim.common.styling import Uri -from gajim.common.styling import URI_RX -from gajim.common.styling import ADDRESS_RX +from gajim.common.styling import Hyperlink +from gajim.common.styling import process_uris +from gajim.common.text_helpers import jid_to_iri
STYLING = { @@ -243,7 +243,7 @@ 'input': 'some kind of link http://foo.com/blah_blah', 'tokens': [ PlainBlock(start=0, end=42, text='some kind of link http://foo.com/blah_blah', spans=[], uris=[ - Uri(start=18, start_byte=18, end=42, end_byte=42, text='http://foo.com/blah_blah') + Hyperlink(start=18, start_byte=18, end=42, end_byte=42, text='http://foo.com/blah_blah', uri='http://foo.com/blah_blah') ]) ] }, @@ -252,7 +252,7 @@ 'input': 'some kind of link http://foo.com/blah_blah,', 'tokens': [ PlainBlock(start=0, end=43, text='some kind of link http://foo.com/blah_blah,', spans=[], uris=[ - Uri(start=18, start_byte=18, end=42, end_byte=42, text='http://foo.com/blah_blah') + Hyperlink(start=18, start_byte=18, end=42, end_byte=42, text='http://foo.com/blah_blah', uri='http://foo.com/blah_blah') ]) ] }, @@ -263,7 +263,7 @@ PlainBlock(start=0, end=44, text='some *kind* of link http://foo.com/blah_blah', spans=[ StrongSpan(start=5, start_byte=5, end=11, end_byte=11, text='*kind*') ], uris=[ - Uri(start=20, start_byte=20, end=44, end_byte=44, text='http://foo.com/blah_blah') + Hyperlink(start=20, start_byte=20, end=44, end_byte=44, text='http://foo.com/blah_blah', uri='http://foo.com/blah_blah') ]) ] }, @@ -272,8 +272,8 @@ 'input': 'some http://foo.com/blah_blah and http://foo.com/blah_blah/123', 'tokens': [ PlainBlock(start=0, end=62, text='some http://foo.com/blah_blah and http://foo.com/blah_blah/123', spans=[], uris=[ - Uri(start=5, start_byte=5, end=29, end_byte=29, text='http://foo.com/blah_blah'), - Uri(start=34, start_byte=34, end=62, end_byte=62, text='http://foo.com/blah_blah/123') + Hyperlink(start=5, start_byte=5, end=29, end_byte=29, text='http://foo.com/blah_blah', uri='http://foo.com/blah_blah'), + Hyperlink(start=34, start_byte=34, end=62, end_byte=62, text='http://foo.com/blah_blah/123', uri='http://foo.com/blah_blah/123') ]) ] }, @@ -281,7 +281,24 @@ }
-URLS = [ +# Most of the URI/JID test sets belong in test_regex.py, and should be imported +# here somehow (TODO). +URIS = [ + 'a:b', + 'a-:b', + 'a.:b', + 'xmpp:conference.gajim.org' + 'xmpp:asd@at', + 'xmpp:asd@asd.at', + 'xmpp:asd-asd@asd.asdasd.at.', + 'xmpp:me@%5B::1%5D', + 'xmpp:myself@127.13.42.69', + 'xmpp:myself@127.13.42.69/localhost', + 'xmpp:%23room%25irc.example@biboumi.xmpp.example', + 'xmpp:+15551234567@cheogram.com', + 'xmpp:romeo@montague.net?message;subject=Test%20Message;body=Here%27s%20a%20test%20message', + + # These seem to be from https://mathiasbynens.be/demo/url-regex 'http://foo.com/blah_blah', 'http://foo.com/blah_blah/', 'http://foo.com/blah_blah_(wikipedia)', @@ -318,83 +335,204 @@ 'http://1337.net', 'http://a.b-c.de', 'http://223.255.255.254', + 'https://foo_bar.example.com/', + + # These are from https://rfc-editor.org/rfc/rfc3513#section-2.2 + 'http://%5BFEDC:BA98:7654:3210:FEDC:BA98:7654:3210]', + 'http://%5B1080:0:0:0:8:800:200C:417A]', + 'http://%5B1080:0:0:0:8:800:200C:417A]', + 'http://%5BFF01:0:0:0:0:0:0:101]', + 'http://%5B0:0:0:0:0:0:0:1]', + 'http://%5B0:0:0:0:0:0:0:0]', + 'http://%5B1080::8:800:200C:417A]', + 'http://%5BFF01::101]', + 'http://%5B::1]', + 'http://[::]', + 'http://%5B0:0:0:0:0:0:13.1.68.3]', + 'http://%5B0:0:0:0:0:FFFF:129.144.52.38]', + 'http://%5B::13.1.68.3]', + 'http://%5B::FFFF:129.144.52.38]', + + # These are from https://rfc-editor.org/rfc/rfc3986#section-1.1.2 + 'ftp://ftp.is.co.za/rfc/rfc1808.txt', + 'http://www.ietf.org/rfc/rfc2396.txt', + 'ldap://[2001:db8::7]/c=GB?objectClass?one', + 'mailto:John.Doe@example.com', + 'news:comp.infosystems.www.servers.unix', + 'tel:+1-816-555-1212', + 'telnet://192.0.2.16:80/', + 'urn:oasis:names:specification:docbook:dtd:xml:4.1.2', +] + + +# * non-URI foos +# * non-absolute URIs +NONURIS = [ + '', + ' ', + '.', + ':', + + # These are from https://mathiasbynens.be/demo/url-regex + '//', + '//a', + '///a', + '///', + 'foo.com', + 'http://foo.bar?q=Spaces should be encoded', + 'http:// shouldfail.com', + ':// should fail', + 'http://foo.bar/foo(bar)baz quux', ]
-EMAILS = [ +# * valid scheme-only URIs +# * valid generic URIs that fail requirements of their specific scheme. +UNACCEPTABLE_URIS = [ + 'scheme:', + + # These are from https://mathiasbynens.be/demo/url-regex + 'http://', + 'http://?', + 'http://??', + 'http://??/', + 'http://#', + 'http://##', + 'http://##/', + 'http:///a', +] + + +JIDS = [ + 'asd@at', 'asd@asd.at', 'asd@asd.asd.at', 'asd@asd.asd-asd.at', 'asd.asd@asd.asd-asd.at', 'asd-asd@asd.asdasd.at', - 'mailto:foo@bar.com.uk', - 'mailto:foo.foo.foo@bar.com.uk', + 'asd-asd@asd.asdasd.at.', + 'me@[::1]', + 'myself@127.13.42.69', + '#room%irc.example@biboumi.xmpp.example', + '+15551234567@cheogram.com', + + # These are from https://rfc-editor.org/rfc/rfc7622#section-3.5 + 'fußball@example.com', + 'π@example.com', + + # These are from https://xmpp.org/extensions/xep-0106.html#examples + r'space\20cadet@example.com', + r'call\20me\20\22ishmael\22@example.com', + r'at\26t\20guy@example.com', + r'd\27artagnan@example.com', + r'\2f.fanboy@example.com', + r'\3a\3afoo\3a\3a@example.com', + r'\3cfoo\3e@example.com', + r'user\40host@example.com', + r'c\3a\net@example.com', + r'c\3a\net@example.com', + r'c\3a\cool\20stuff@example.com', + r'c\3a\5c5commas@example.com', + r'here\27s_a_wild_\26_\2fcr%zy\2f_address@example.com', + r'here\27s_a_wild_\26_\2fcr%zy\2f_address_for\3a\3cwv\3e(\22IMPS\22)@example.com', + # Some more from the same document + r'tréville\40musketeers.lit@smtp.gascon.fr', + r'\5c3and\2is\5c5cool@example.com', + r'CN=D\27Artagnan\20Saint-Andr\E9,O=Example\20\26\20Company,\20Inc.,DC=example,DC=com@st.example.com', + r'somenick!user\22\26\27\2f\3a\3c\3e\5c3address@example.com', + + # https://en.wikipedia.org/wiki/E-mail_address#Internationalization_examples + # Do note that these are *e-mail* addresses and might not all be vaild JIDs. + 'Pelé@example.com', + 'δοκιμή@παράδειγμα.δοκιμή', + '我買@屋企.香港', + '二ノ宮@黒川.日本', + 'медведь@с-балалайкой.рф', + #'संपर्क@डाटामेल.भारत', fails because of the 2 combining chars in localpart ]
-EMAILS_WITH_TEXT = [ - ('write to my email mailto:foo@bar.com.uk (but not to mailto:bar@foo.com)', - ['mailto:foo@bar.com.uk', 'mailto:bar@foo.com']), - ('write to my email mailtomailto:foo@bar.com.uk (but not to mailto:bar@foo.com)', - ['foo@bar.com.uk', 'mailto:bar@foo.com']), +NONJIDS = [ + '', + '@', + + # These are from https://rfc-editor.org/rfc/rfc7622#section-3.5 + '"juliet"@example.com', + 'foo bar@example.com', # search is expected to find 'bar@example.com' + '@example.com', + 'henryⅣ@example.com', # localpart has a compatibility-decomposable cp + '♚@example.com', # localpart has a symbol cp + 'juliet@', ]
-URL_WITH_TEXT = [ - ('see this http://userid@example.com/ link', 'http://userid@example.com/'), - ('see this http://userid@example.com/, and ..', 'http://userid@example.com/'), - ('http://userid@example.com/', 'http://userid@example.com/'), -] - -XMPP_URIS = [ - ('see xmpp:romeo@montague.net?message;subject=Test%20Message;body=Here%27s%20a%20test%20message ...', 'xmpp:romeo@montague.net?message;subject=Test%20Message;body=Here%27s%20a%20test%20message'), +URIS_WITH_TEXT = [ + ('write to my email mailto:foo@bar.com.uk (but not to mailto:bar@foo.com)', + ['mailto:foo@bar.com.uk', 'mailto:bar@foo.com']), + ('write to my email mailtomailto:foo@bar.com.uk (but not to mailto:bar@foo.com)', + ['mailtomailto:foo@bar.com.uk', 'mailto:bar@foo.com']), + ('see this http://userid@example.com/ link', ['http://userid@example.com/']), + ('see this http://userid@example.com/, and ..', ['http://userid@example.com/']), + ('http://userid@example.com/', ['http://userid@example.com/']), + ('"http://userid@example.com/"', ['http://userid@example.com/']), + ('regexes are useless (see https://en.wikipedia.org/wiki/Recursion_(computer_science)), but comfy', ['https://en.wikipedia.org/wiki/Recursion_(computer_science)']), ]
class Test(unittest.TestCase): + @staticmethod + def wrap(link: str) -> str: + return f'Prologue (link: {link}), and epilogue!' + def test_styling(self): for _name, params in STYLING.items(): result = styling.process(params['input']) - self.assertTrue(result.blocks == params['tokens']) - - def test_urls(self): - for url in URLS: - match = URI_RX.search(url) - self.assertIsNotNone(match) - start = match.start() - end = match.end() - self.assertTrue(url[start:end] == url) - - def test_emails(self): - for email in EMAILS: - match = ADDRESS_RX.search(email) - self.assertIsNotNone(match) - start = match.start() - end = match.end() - self.assertTrue(email[start:end] == email) - - def test_emails_with_text(self): - for text, result in EMAILS_WITH_TEXT: - match = ADDRESS_RX.findall(text) - self.assertIsNotNone(match) + self.assertEqual(result.blocks, params['tokens']) + + def test_uris(self): + for uri in URIS: + text = self.wrap(uri) + hlinks = process_uris(text) + self.assertEqual(len(hlinks), 1, text) + self.assertEqual(hlinks[0].uri, uri, text) + + def test_invalid_uris(self): + for foo in NONURIS + UNACCEPTABLE_URIS: + if foo == 'scheme:' or\ + foo == 'http://': + continue # these currently fial (FIXME) + text = self.wrap(foo) + hlinks = process_uris(text) + if len(hlinks) == 0: + continue + self.assertEqual(len(hlinks), 1, text) + self.assertNotEqual(hlinks[0].text, foo, text) + + def test_jids(self): + for jid in JIDS: + text = self.wrap(jid) + hlinks = process_uris(text) + self.assertEqual(len(hlinks), 1, text) + self.assertEqual(hlinks[0].text, jid, text) + self.assertEqual(hlinks[0].uri, jid_to_iri(jid), text) + + def test_nonjids(self): + for foo in NONJIDS: + if foo == 'juliet@': + continue # this currently fials (FIXME) + text = self.wrap(foo) + hlinks = process_uris(text) + if len(hlinks) == 0: + continue + self.assertEqual(len(hlinks), 1, text) + self.assertNotEqual(hlinks[0].text, foo, text) + + def test_uris_with_text(self): + for text, result in URIS_WITH_TEXT: + hlinks = process_uris(text) + self.assertEqual(len(hlinks), len(result), text) for i, res in enumerate(result): - self.assertTrue(match[i][0] == res) - - def test_url_with_text(self): - for text, result in URL_WITH_TEXT: - match = URI_RX.search(text) - self.assertIsNotNone(match) - start = match.start() - end = match.end() - self.assertTrue(text[start:end] == result) - - def test_xmpp_uris(self): - for text, result in XMPP_URIS: - match = ADDRESS_RX.search(text) - self.assertIsNotNone(match) - start = match.start() - end = match.end() - self.assertTrue(text[start:end] == result) + self.assertEqual(hlinks[i].text, res, text)
if __name__ == "__main__":
===================================== test/no_gui/test_text_helpers.py ===================================== @@ -0,0 +1,35 @@ +import unittest + +from gajim.common.text_helpers import escape_iri_path_segment +from gajim.common.text_helpers import jid_to_iri + + +class Test(unittest.TestCase): + def test_escape_iri_path_segment(self): + self.assertEqual(escape_iri_path_segment(''), '', '<empty string>') + + über = 'u\u0308ber' + self.assertEqual(escape_iri_path_segment(über), über) + + self.assertEqual( + escape_iri_path_segment(''.join(chr(c) for c in range(0x20,0x7F))), + '%20!%22%23$%25&'()*+,-.%2F0123456789:;%3C=%3E%3F@ABCDEFGHIJKLMN' + 'OPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~', + 'ASCII printable') + + self.assertEqual(escape_iri_path_segment( + ''.join(chr(c) for c in range(0x01,0x20)) + chr(0x7F)), + ''.join('%%%02X'%c for c in range(0x01,0x20)) + '%7F', + 'ASCII control (no null)') + + + def test_jid_to_iri(self): + jid = r'foo@bar' + self.assertEqual(jid_to_iri(jid), fr'xmpp:{jid}', jid) + jid = r'my\20self@[::1]/home' + self.assertEqual(jid_to_iri(jid), + r'xmpp:my%5C20self@%5B::1%5D/home', jid) + + +if __name__ == '__main__': + unittest.main()
View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/1ef1e8fecf4517d3234ffa98b3e0a9245...
-
Philipp Hörist (＠lovetox)