[Git][gajim/gajim][master] fix: Styling: Various link parsing issues

24 Oct 2022


      Philipp Hörist pushed to branch master at gajim / gajim
Commits:
1ef1e8fe by mjk at 2022-10-24T17:50:41+00:00
fix: Styling: Various link parsing issues
Also, a small feature: tooltip for JIDs that shows the xmpp URI.
Regression: words starting with &#39;www.&#39; are no longer linkified.
Fixes #11218, #11144
- - - - -
8 changed files:
- gajim/common/regex.py
- gajim/common/styling.py
- + gajim/common/text_helpers.py
- gajim/gtk/conversation/plain_widget.py
- setup.cfg
- test/no_gui/test_regex.py
- test/no_gui/test_styling.py
- + test/no_gui/test_text_helpers.py
Changes:
=====================================
gajim/common/regex.py
=====================================
@@ -13,6 +13,7 @@
 # along with Gajim. If not, see http://www.gnu.org/licenses/.
import re
+from typing import Any
INVALID_XML_CHARS_REGEX = re.compile(
@@ -21,3 +22,101 @@
 # at least one character in 3 parts (before @, after @, after .)
 STH_AT_STH_DOT_STH_REGEX = re.compile(
     r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+.[a-zA-Z0-9-.]+$')
+
+
+# Universal/Internationalized Resource Identifiers and their components, as
+# defined in [ABNF for IRI References and IRIs]
+# https://rfc-editor.org/rfc/rfc3987#section-2.2 and
+# [Representing IPv6 Zone Identifiers in Address Literals and URIs]
+# https://www.rfc-editor.org/rfc/rfc6874.
+ALPHA          = r'[A-Za-z]'
+HEXDIG         = r'[0-9A-Fa-f]'
+sub_delims     = r"!$&'()*+,;="
+ucschar        = r'\xA0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF'\
+    r'\U00010000-\U0001FFFD\U00020000-\U0002FFFD\U00030000-\U0003FFFD'\
+    r'\U00040000-\U0004FFFD\U00050000-\U0005FFFD\U00060000-\U0006FFFD'\
+    r'\U00070000-\U0007FFFD\U00080000-\U0008FFFD\U00090000-\U0009FFFD'\
+    r'\U000A0000-\U000AFFFD\U000B0000-\U000BFFFD\U000C0000-\U000CFFFD'\
+    r'\U000D0000-\U000DFFFD\U000E1000-\U000EFFFD'
+iprivate       = r'\uE000-\uF8FF'\
+    r'\U000F0000-\U000FFFFD\U00100000-\U0010FFFD'
+unreserved     = r'A-Za-z0-9-._~'
+iunreserved    = fr'{unreserved}{ucschar}'
+pct_encoded    = fr'%{HEXDIG}{HEXDIG}'
+ipchar         = fr'(?:[{iunreserved}{sub_delims}:@]|{pct_encoded})'
+iuserinfo      = fr'(?:[{iunreserved}{sub_delims}:]|{pct_encoded})*'
+ireg_name      = fr'(?:[{iunreserved}{sub_delims}]|{pct_encoded})*'
+dec_octet      = r'(?:25[0-5]'\
+                 r'|2[0-4][0-9]'\
+                 r'|1[0-9][0-9]'\
+                 r'|[1-9][0-9]'\
+                 r'|[0-9])'
+IPv4address    = fr'{dec_octet}(?:.{dec_octet}){{3}}'
+h16            = fr'{HEXDIG}{{1,4}}'
+ls32           = fr'(?:{h16}:{h16}|{IPv4address})'
+IPv6address    =\
+                              fr'(?:(?:{h16}:){{6}}{ls32}'\
+                              fr'|::(?:{h16}:){{5}}{ls32}'\
+                    fr'|(?:{h16})?::(?:{h16}:){{4}}{ls32}'\
+    fr'|(?:(?:{h16}:){{,1}}{h16})?::(?:{h16}:){{3}}{ls32}'\
+    fr'|(?:(?:{h16}:){{,2}}{h16})?::(?:{h16}:){{2}}{ls32}'\
+    fr'|(?:(?:{h16}:){{,3}}{h16})?::{h16}:{ls32}'\
+    fr'|(?:(?:{h16}:){{,4}}{h16})?::{ls32}'\
+    fr'|(?:(?:{h16}:){{,5}}{h16})?::{h16}'\
+    fr'|(?:(?:{h16}:){{,6}}{h16})?::)'  # noqa: E126,E127,E131
+ZoneID         = fr'(?:{unreserved}|{pct_encoded})+'
+IPv6addrz      = fr'{IPv6address}%25{ZoneID}'
+IPvFuture      = fr'[Vv]{HEXDIG}+.[{unreserved}{sub_delims}:]+'
+IP_literal     = fr'[(?:{IPv6address}|{IPv6addrz}|{IPvFuture})]'
+# ihost        = fr'(?P<host>{IP_literal}|{IPv4address}|{ireg_name})'
+# The below is equivalent to the above for the purpose of validation, but
+# better for the purpose of search: e.g., matches 'foo://1.2.3.4.5' completely.
+ihost          = fr'(?P<host>{IP_literal}|{ireg_name})'
+port           = r'[0-9]*'
+iauthority     = fr'(?:{iuserinfo}@)?{ihost}(?::{port})?'
+isegment       = fr'{ipchar}*'
+isegment_nz    = fr'{ipchar}+'
+ipath_abempty  = fr'(?:/{isegment})*'
+ipath_absolute = fr'/(?:{isegment_nz}(?:/{isegment})*)?'
+ipath_rootless = fr'{isegment_nz}(?:/{isegment})*'
+ipath_empty    = r''
+ihier_part     = fr'(?://{iauthority}{ipath_abempty}'\
+                 fr'|{ipath_absolute}'\
+                 fr'|{ipath_rootless}'\
+                 fr'|{ipath_empty})'
+iquery         = fr'(?:{ipchar}|[/?{iprivate}])*'
+ifragment      = fr'(?:{ipchar}|[/?])*'
+scheme         = fr'(?P<scheme>{ALPHA}[a-zA-Z0-9+.-]*)'
+IRI            = fr'{scheme}:{ihier_part}(?:?{iquery})?(?:#{ifragment})?'
+
+
+# https://rfc-editor.org/rfc/rfc7564#section-4.2
+class PRECIS:
+    LetterDigits = r'\w'  # roughly (e.g., includes '_' as an extra)
+    Spaces       = r'\s'  # roughly?
+    Symbols      = r'♚'   # let's make that test pass!
+    HasCompat    = r'Ⅳ'  # and that one!
+    IdentifierClass: Any
+
+
+class PRECIS_IdentifierClass:
+    _p = PRECIS
+    Valid      = fr'{_p.LetterDigits}\x21-\x7e'
+    Disallowed = fr'{_p.Spaces}{_p.Symbols}{_p.HasCompat}'  # there's a ton more
+
+
+PRECIS.IdentifierClass = PRECIS_IdentifierClass
+
+
+# (Rough approximation of) Jabber IDs, as defined in [XMPP: Address Format]
+# https://rfc-editor.org/rfc/rfc7622#section-3.
+class XMPP:
+    _id = PRECIS.IdentifierClass
+    _excl      = '"&'/:<>@'  # <#section-3.3.1>
+    localpart  = fr'(?:(?![{_excl}{_id.Disallowed}])[{_id.Valid}])+'
+    # ^ doesn't take into account "contextual rules" <rfc7564#section-4.2.2>
+    ifqdn      = fr'[{iunreserved}{sub_delims}]+'
+    # ^ probably correct
+    domainpart = fr'(?:{IP_literal}|{IPv4address}|{ifqdn})'
+    jid        = fr'(?P<local>{localpart})@(?P<domain>{domainpart})'
+    # ^ notably, doesn't include resourcepart (TODO)
=====================================
gajim/common/styling.py
=====================================
@@ -25,6 +25,8 @@
from gi.repository import GLib
+from gajim.common import regex
+from gajim.common.text_helpers import jid_to_iri
 from gajim.gui.emoji_data import emoji_data
PRE = '`'
@@ -44,13 +46,7 @@
 BLOCK_NESTED_RX = re.compile(PRE_NESTED_RX + '|' + QUOTE_RX, re.S | re.M)
 UNQUOTE_RX = re.compile(r'^> |^>', re.M)
-URI_RX = r'(?P<uri>([\w-]+://|www[.])[\S()<>]+?(?=[,>]?(\s|\Z)+))'
-ADDRESS_RX = r'(?P<address>\b((xmpp|mailto):)?[\w-]+(.[\w-]+)*@([\w-]*?.)+[\w]+([?].*?(?=([\s),]|$)))?)'  # noqa: E501
-URI_ADDRESS_RX = URI_RX + '|' + ADDRESS_RX
-
-URI_RX = re.compile(URI_RX)
-ADDRESS_RX = re.compile(ADDRESS_RX)
-URI_ADDRESS_RX = re.compile(URI_ADDRESS_RX)
+URI_OR_JID_RX = re.compile(fr'(?P<uri>{regex.IRI})|(?P<jid>{regex.XMPP.jid})')
EMOJI_RX = emoji_data.get_regex()
 EMOJI_RX = re.compile(EMOJI_RX)
@@ -68,33 +64,31 @@ class StyleObject:
@dataclass
-class BaseUri(StyleObject):
+class BaseHyperlink(StyleObject):
     name: str
+    uri: str
     start_byte: int
     end_byte: int
def get_markup_string(self) -> str:
+        href = GLib.markup_escape_text(self.uri)
         text = GLib.markup_escape_text(self.text)
-        return f'<a href="{text}">{text}</a>'
+        title = '' if href == text else f' title="{href}"'
+        return f'<a href="{href}"{title}>{text}</a>'
@dataclass
-class Uri(BaseUri):
+class Hyperlink(BaseHyperlink):
     name: str = field(default='uri', init=False)
@dataclass
-class Address(BaseUri):
-    name: str = field(default='address', init=False)
-
-
-@dataclass
-class XMPPAddress(BaseUri):
+class XMPPAddress(BaseHyperlink):
     name: str = field(default='xmppadr', init=False)
@dataclass
-class MailAddress(BaseUri):
+class MailAddress(BaseHyperlink):
     name: str = field(default='mailadr', init=False)
@@ -117,7 +111,7 @@ def from_match(cls, match: Match[str]) -> Block:
 class PlainBlock(Block):
     name: str = field(default='plain', init=False)
     spans: list[Span] = field(default_factory=list)
-    uris: list[BaseUri] = field(default_factory=list)
+    uris: list[BaseHyperlink] = field(default_factory=list)
     emojis: list[Emoji] = field(default_factory=list)
@classmethod
@@ -224,11 +218,11 @@ def process(text: Union[str, bytes], level: int = 0) -> ParsingResult:
     return ParsingResult(text, blocks)
-def process_uris(text: Union[str, bytes]) -> list[BaseUri]:
+def process_uris(text: Union[str, bytes]) -> list[BaseHyperlink]:
     if isinstance(text, bytes):
         text = text.decode()
-    uris: list[BaseUri] = []
+    uris: list[BaseHyperlink] = []
     offset = 0
     offset_bytes = 0
     for line in text.splitlines(keepends=True):
@@ -330,20 +324,36 @@ def _parse_line(line: str, offset: int, offset_bytes: int) -> list[Span]:
     return spans
-def _parse_uris(line: str, offset: int, offset_bytes: int) -> list[BaseUri]:
-    uris: list[BaseUri] = []
-
-    def make(match: Match[str], is_address: bool) -> BaseUri:
-        return _make_link(line,
-                          match.start(),
-                          match.end() - 1,
-                          offset,
-                          offset_bytes,
-                          is_address)
-
-    for match in URI_ADDRESS_RX.finditer(line):
-        uri = make(match, bool(match.group('address')))
-        uris.append(uri)
+def _parse_uris(line: str,
+                offset: int,
+                offset_bytes: int) -> list[BaseHyperlink]:
+    uris: list[BaseHyperlink] = []
+
+    def make(start: int, end: int, is_jid: bool) -> BaseHyperlink:
+        if line[end - 1] == ',':
+            # Trim one trailing comma
+            end -= 1
+        if '(' in line[:start] and line[end - 1] == ')':
+            # Trim one trailing closing parenthesis if the match is preceded
+            # by an opening one somewhere on the line
+            end -= 1
+        return _make_hyperlink(line,
+                               start,
+                               end - 1,
+                               offset,
+                               offset_bytes,
+                               is_jid)
+
+    for match in URI_OR_JID_RX.finditer(line):
+        start, end = match.span()
+        if match.group('jid') is not None:
+            uris.append(make(start, end, True))
+        elif not re.fullmatch('[^:]+:', line[start:end]) and\
+            (not re.fullmatch('https?', match.group('scheme'), re.IGNORECASE) or
+             match.group('host')):
+            # URIs that consist only of a scheme are thusly excluded.
+            # http(s) URIs without host or with empty host are excluded too.
+            uris.append(make(start, end, False))
return uris
@@ -407,12 +417,12 @@ def _make_span(line: str,
                       text=text)
-def _make_link(line: str,
-               start: int,
-               end: int,
-               offset: int,
-               offset_bytes: int,
-               is_address: bool) -> BaseUri:
+def _make_hyperlink(line: str,
+                    start: int,
+                    end: int,
+                    offset: int,
+                    offset_bytes: int,
+                    is_jid: bool) -> BaseHyperlink:
text = line[start:end + 1]
@@ -422,22 +432,22 @@ def _make_link(line: str,
     start += offset
     end += offset + 1
-    if not is_address:
-        cls_ = Uri
-
-    elif text.startswith('xmpp'):
+    uri = text
+    if is_jid:
         cls_ = XMPPAddress
-
-    elif text.startswith('mailto'):
+        uri = jid_to_iri(text)
+    elif re.match('xmpp:', text, re.IGNORECASE):
+        cls_ = XMPPAddress
+    elif re.match('mailto:', text, re.IGNORECASE):
         cls_ = MailAddress
-
     else:
-        cls_ = Address
+        cls_ = Hyperlink
return cls_(start=start,
                 start_byte=start_byte,
                 end=end,
                 end_byte=end_byte,
+                uri=uri,
                 text=text)
=====================================
gajim/common/text_helpers.py
=====================================
@@ -0,0 +1,33 @@
+# This file is part of Gajim.
+#
+# Gajim is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published
+# by the Free Software Foundation; version 3 only.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see http://www.gnu.org/licenses/.
+
+from gi.repository import GLib
+from gajim.common import regex
+
+# from RFC 3986, 3.3. Path (pchar without unreserved and pct-encoded):
+_reserved_chars_allowed_in_path_segment = regex.sub_delims + ':@'
+
+
+def escape_iri_path_segment(s: str) -> str:
+    return GLib.Uri.escape_string(
+        s, _reserved_chars_allowed_in_path_segment, True)
+
+
+def escape_iri_path(s: str) -> str:
+    return GLib.Uri.escape_string(
+        s, _reserved_chars_allowed_in_path_segment + '/', True)
+
+
+def jid_to_iri(jid: str) -> str:
+    return 'xmpp:' + escape_iri_path(jid)
=====================================
gajim/gtk/conversation/plain_widget.py
=====================================
@@ -31,7 +31,7 @@
 from gajim.common.helpers import open_uri
 from gajim.common.helpers import parse_uri
 from gajim.common.structs import URI
-from gajim.common.styling import BaseUri
+from gajim.common.styling import BaseHyperlink
 from gajim.common.styling import PlainBlock
 from gajim.common.styling import process_uris
@@ -44,7 +44,7 @@
log = logging.getLogger('gajim.gui.conversaion.plain_widget')
-URI_TAGS = ['uri', 'address', 'xmppadr', 'mailadr']
+URI_TAGS = ['uri', 'xmppadr', 'mailadr']
 STYLE_TAGS = ['strong', 'emphasis', 'strike', 'pre']
@@ -106,7 +106,7 @@ def _on_populate_popup(self, label: Gtk.Label, menu: Gtk.Menu) -> None:
         menu.prepend(action_menu_item)
         menu.show_all()
-    def _build_link_markup(self, text: str, uris: list[BaseUri]) -> str:
+    def _build_link_markup(self, text: str, uris: list[BaseHyperlink]) -> str:
         markup_text = ''
         after = GLib.markup_escape_text(text.strip())
         for uri in uris:
=====================================
setup.cfg
=====================================
@@ -9,6 +9,7 @@ exclude =
     typings,
     gajim/common/config.py,
     gajim/common/optparser.py,
+    gajim/common/regex.py,
     gajim/common/socks5.py,
     gajim/gui,
 max-complexity = 15
=====================================
test/no_gui/test_regex.py
=====================================
@@ -1,27 +1,38 @@
 from typing import Any
import unittest
+import re
-from gajim.common.styling import URI_RX
-from gajim.common.styling import ADDRESS_RX
+from gajim.common import regex
TEST_URIS = [
+    'scheme:',
+    'http:///',
+    'http://',
     'http://google.com/',
     'http://google.com',
     'http://www.google.ca/search?q=xmpp',
     'http://tools.ietf.org/html/draft-saintandre-rfc3920bis-05#section-12.3',
     'http://en.wikipedia.org/wiki/Protocol_(computing)',
     'http://en.wikipedia.org/wiki/Protocol_%28computing%29',
-]
-
-ADDRESSES = [
     'mailto:test@example.org',
     'xmpp:example-node@example.com',
-    # 'xmpp:example-node@example.com/some-resource', FIXME
+    'xmpp:example-node@example.com/some-resource',
     'xmpp:example-node@example.com?message',
 ]
+JIDS = [
+    'example-node@example.com',
+    'me@[::1]',
+    'myself@127.13.42.69',
+    #'myself@127.13.42.69/localhost',  TODO
+    '#room%irc.example@biboumi.xmpp.example',
+    r'here\27s_a_wild_\26_\2fcr%zy\2f_address_for\3a\3cwv\3e(\22IMPS\22)@example.com',
+    r'CN=D\27Artagnan\20Saint-Andr\E9,O=Example\20\26\20Company,\20Inc.,DC=example,DC=com@st.example.com',
+    'δοκιμή@παράδειγμα.δοκιμή',
+]
+
class Test(unittest.TestCase):
     def test_links_regexp_entire(self):
@@ -34,10 +45,10 @@ def assert_matches_all(regex: Any, uri_str: str) -> None:
             self.assertEqual(m.span(), str_span, uri_str)
for uri in TEST_URIS:
-            assert_matches_all(URI_RX, uri)
+            assert_matches_all(re.compile(regex.IRI), uri)
-        for addr in ADDRESSES:
-            assert_matches_all(ADDRESS_RX, addr)
+        for addr in JIDS:
+            assert_matches_all(re.compile(regex.XMPP.jid), addr)
if __name__ == '__main__':
=====================================
test/no_gui/test_styling.py
=====================================
@@ -8,9 +8,9 @@
 from gajim.common.styling import StrongSpan
 from gajim.common.styling import EmphasisSpan
 from gajim.common.styling import StrikeSpan
-from gajim.common.styling import Uri
-from gajim.common.styling import URI_RX
-from gajim.common.styling import ADDRESS_RX
+from gajim.common.styling import Hyperlink
+from gajim.common.styling import process_uris
+from gajim.common.text_helpers import jid_to_iri
STYLING = {
@@ -243,7 +243,7 @@
         'input': 'some kind of link http://foo.com/blah_blah',
         'tokens': [
             PlainBlock(start=0, end=42, text='some kind of link http://foo.com/blah_blah', spans=[], uris=[
-                Uri(start=18, start_byte=18, end=42, end_byte=42, text='http://foo.com/blah_blah')
+                Hyperlink(start=18, start_byte=18, end=42, end_byte=42, text='http://foo.com/blah_blah', uri='http://foo.com/blah_blah')
             ])
         ]
     },
@@ -252,7 +252,7 @@
         'input': 'some kind of link http://foo.com/blah_blah,',
         'tokens': [
             PlainBlock(start=0, end=43, text='some kind of link http://foo.com/blah_blah,', spans=[], uris=[
-                Uri(start=18, start_byte=18, end=42, end_byte=42, text='http://foo.com/blah_blah')
+                Hyperlink(start=18, start_byte=18, end=42, end_byte=42, text='http://foo.com/blah_blah', uri='http://foo.com/blah_blah')
             ])
         ]
     },
@@ -263,7 +263,7 @@
             PlainBlock(start=0, end=44, text='some *kind* of link http://foo.com/blah_blah', spans=[
                 StrongSpan(start=5, start_byte=5, end=11, end_byte=11, text='*kind*')
             ], uris=[
-                Uri(start=20, start_byte=20, end=44, end_byte=44, text='http://foo.com/blah_blah')
+                Hyperlink(start=20, start_byte=20, end=44, end_byte=44, text='http://foo.com/blah_blah', uri='http://foo.com/blah_blah')
             ])
         ]
     },
@@ -272,8 +272,8 @@
         'input': 'some http://foo.com/blah_blah and http://foo.com/blah_blah/123',
         'tokens': [
             PlainBlock(start=0, end=62, text='some http://foo.com/blah_blah and http://foo.com/blah_blah/123', spans=[], uris=[
-                Uri(start=5, start_byte=5, end=29, end_byte=29, text='http://foo.com/blah_blah'),
-                Uri(start=34, start_byte=34, end=62, end_byte=62, text='http://foo.com/blah_blah/123')
+                Hyperlink(start=5, start_byte=5, end=29, end_byte=29, text='http://foo.com/blah_blah', uri='http://foo.com/blah_blah'),
+                Hyperlink(start=34, start_byte=34, end=62, end_byte=62, text='http://foo.com/blah_blah/123', uri='http://foo.com/blah_blah/123')
             ])
         ]
     },
@@ -281,7 +281,24 @@
 }
-URLS = [
+# Most of the URI/JID test sets belong in test_regex.py, and should be imported
+# here somehow (TODO).
+URIS = [
+    'a:b',
+    'a-:b',
+    'a.:b',
+    'xmpp:conference.gajim.org'
+    'xmpp:asd@at',
+    'xmpp:asd@asd.at',
+    'xmpp:asd-asd@asd.asdasd.at.',
+    'xmpp:me@%5B::1%5D',
+    'xmpp:myself@127.13.42.69',
+    'xmpp:myself@127.13.42.69/localhost',
+    'xmpp:%23room%25irc.example@biboumi.xmpp.example',
+    'xmpp:+15551234567@cheogram.com',
+    'xmpp:romeo@montague.net?message;subject=Test%20Message;body=Here%27s%20a%20test%20message',
+
+    # These seem to be from https://mathiasbynens.be/demo/url-regex
     'http://foo.com/blah_blah',
     'http://foo.com/blah_blah/',
     'http://foo.com/blah_blah_(wikipedia)',
@@ -318,83 +335,204 @@
     'http://1337.net',
     'http://a.b-c.de',
     'http://223.255.255.254',
+
     'https://foo_bar.example.com/',
+
+    # These are from https://rfc-editor.org/rfc/rfc3513#section-2.2
+    'http://%5BFEDC:BA98:7654:3210:FEDC:BA98:7654:3210]',
+    'http://%5B1080:0:0:0:8:800:200C:417A]',
+    'http://%5B1080:0:0:0:8:800:200C:417A]',
+    'http://%5BFF01:0:0:0:0:0:0:101]',
+    'http://%5B0:0:0:0:0:0:0:1]',
+    'http://%5B0:0:0:0:0:0:0:0]',
+    'http://%5B1080::8:800:200C:417A]',
+    'http://%5BFF01::101]',
+    'http://%5B::1]',
+    'http://[::]',
+    'http://%5B0:0:0:0:0:0:13.1.68.3]',
+    'http://%5B0:0:0:0:0:FFFF:129.144.52.38]',
+    'http://%5B::13.1.68.3]',
+    'http://%5B::FFFF:129.144.52.38]',
+
+    # These are from https://rfc-editor.org/rfc/rfc3986#section-1.1.2
+    'ftp://ftp.is.co.za/rfc/rfc1808.txt',
+    'http://www.ietf.org/rfc/rfc2396.txt',
+    'ldap://[2001:db8::7]/c=GB?objectClass?one',
+    'mailto:John.Doe@example.com',
+    'news:comp.infosystems.www.servers.unix',
+    'tel:+1-816-555-1212',
+    'telnet://192.0.2.16:80/',
+    'urn:oasis:names:specification:docbook:dtd:xml:4.1.2',
+]
+
+
+# * non-URI foos
+# * non-absolute URIs
+NONURIS = [
+    '',
+    ' ',
+    '.',
+    ':',
+
+    # These are from https://mathiasbynens.be/demo/url-regex
+    '//',
+    '//a',
+    '///a',
+    '///',
+    'foo.com',
+    'http://foo.bar?q=Spaces should be encoded',
+    'http:// shouldfail.com',
+    ':// should fail',
+    'http://foo.bar/foo(bar)baz quux',
 ]
-EMAILS = [
+# * valid scheme-only URIs
+# * valid generic URIs that fail requirements of their specific scheme.
+UNACCEPTABLE_URIS = [
+    'scheme:',
+
+    # These are from https://mathiasbynens.be/demo/url-regex
+    'http://',
+    'http://?',
+    'http://??',
+    'http://??/',
+    'http://#',
+    'http://##',
+    'http://##/',
+    'http:///a',
+]
+
+
+JIDS = [
+    'asd@at',
     'asd@asd.at',
     'asd@asd.asd.at',
     'asd@asd.asd-asd.at',
     'asd.asd@asd.asd-asd.at',
     'asd-asd@asd.asdasd.at',
-    'mailto:foo@bar.com.uk',
-    'mailto:foo.foo.foo@bar.com.uk',
+    'asd-asd@asd.asdasd.at.',
+    'me@[::1]',
+    'myself@127.13.42.69',
+    '#room%irc.example@biboumi.xmpp.example',
+    '+15551234567@cheogram.com',
+
+    # These are from https://rfc-editor.org/rfc/rfc7622#section-3.5
+    'fußball@example.com',
+    'π@example.com',
+
+    # These are from https://xmpp.org/extensions/xep-0106.html#examples
+    r'space\20cadet@example.com',
+    r'call\20me\20\22ishmael\22@example.com',
+    r'at\26t\20guy@example.com',
+    r'd\27artagnan@example.com',
+    r'\2f.fanboy@example.com',
+    r'\3a\3afoo\3a\3a@example.com',
+    r'\3cfoo\3e@example.com',
+    r'user\40host@example.com',
+    r'c\3a\net@example.com',
+    r'c\3a\net@example.com',
+    r'c\3a\cool\20stuff@example.com',
+    r'c\3a\5c5commas@example.com',
+    r'here\27s_a_wild_\26_\2fcr%zy\2f_address@example.com',
+    r'here\27s_a_wild_\26_\2fcr%zy\2f_address_for\3a\3cwv\3e(\22IMPS\22)@example.com',
+    # Some more from the same document
+    r'tréville\40musketeers.lit@smtp.gascon.fr',
+    r'\5c3and\2is\5c5cool@example.com',
+    r'CN=D\27Artagnan\20Saint-Andr\E9,O=Example\20\26\20Company,\20Inc.,DC=example,DC=com@st.example.com',
+    r'somenick!user\22\26\27\2f\3a\3c\3e\5c3address@example.com',
+
+    # https://en.wikipedia.org/wiki/E-mail_address#Internationalization_examples
+    # Do note that these are *e-mail* addresses and might not all be vaild JIDs.
+    'Pelé@example.com',
+    'δοκιμή@παράδειγμα.δοκιμή',
+    '我買@屋企.香港',
+    '二ノ宮@黒川.日本',
+    'медведь@с-балалайкой.рф',
+    #'संपर्क@डाटामेल.भारत',  fails because of the 2 combining chars in localpart
 ]
-EMAILS_WITH_TEXT = [
-    ('write to my email mailto:foo@bar.com.uk (but not to mailto:bar@foo.com)',
-     ['mailto:foo@bar.com.uk', 'mailto:bar@foo.com']),
-    ('write to my email mailtomailto:foo@bar.com.uk (but not to mailto:bar@foo.com)',
-     ['foo@bar.com.uk', 'mailto:bar@foo.com']),
+NONJIDS = [
+    '',
+    '@',
+
+    # These are from https://rfc-editor.org/rfc/rfc7622#section-3.5
+    '"juliet"@example.com',
+    'foo bar@example.com',  # search is expected to find 'bar@example.com'
+    '@example.com',
+    'henryⅣ@example.com',  # localpart has a compatibility-decomposable cp
+    '♚@example.com',        # localpart has a symbol cp
+    'juliet@',
 ]
-URL_WITH_TEXT = [
-    ('see this http://userid@example.com/ link', 'http://userid@example.com/'),
-    ('see this http://userid@example.com/, and ..', 'http://userid@example.com/'),
-    ('http://userid@example.com/', 'http://userid@example.com/'),
-]
-
-XMPP_URIS = [
-    ('see xmpp:romeo@montague.net?message;subject=Test%20Message;body=Here%27s%20a%20test%20message ...', 'xmpp:romeo@montague.net?message;subject=Test%20Message;body=Here%27s%20a%20test%20message'),
+URIS_WITH_TEXT = [
+    ('write to my email mailto:foo@bar.com.uk (but not to mailto:bar@foo.com)',
+     ['mailto:foo@bar.com.uk', 'mailto:bar@foo.com']),
+    ('write to my email mailtomailto:foo@bar.com.uk (but not to mailto:bar@foo.com)',
+     ['mailtomailto:foo@bar.com.uk', 'mailto:bar@foo.com']),
+    ('see this http://userid@example.com/ link', ['http://userid@example.com/']),
+    ('see this http://userid@example.com/, and ..', ['http://userid@example.com/']),
+    ('http://userid@example.com/', ['http://userid@example.com/']),
+    ('"http://userid@example.com/"', ['http://userid@example.com/']),
+    ('regexes are useless (see https://en.wikipedia.org/wiki/Recursion_(computer_science)), but comfy', ['https://en.wikipedia.org/wiki/Recursion_(computer_science)']),
 ]
class Test(unittest.TestCase):
+    @staticmethod
+    def wrap(link: str) -> str:
+        return f'Prologue (link: {link}), and epilogue!'
+
     def test_styling(self):
         for _name, params in STYLING.items():
             result = styling.process(params['input'])
-            self.assertTrue(result.blocks == params['tokens'])
-
-    def test_urls(self):
-        for url in URLS:
-            match = URI_RX.search(url)
-            self.assertIsNotNone(match)
-            start = match.start()
-            end = match.end()
-            self.assertTrue(url[start:end] == url)
-
-    def test_emails(self):
-        for email in EMAILS:
-            match = ADDRESS_RX.search(email)
-            self.assertIsNotNone(match)
-            start = match.start()
-            end = match.end()
-            self.assertTrue(email[start:end] == email)
-
-    def test_emails_with_text(self):
-        for text, result in EMAILS_WITH_TEXT:
-            match = ADDRESS_RX.findall(text)
-            self.assertIsNotNone(match)
+            self.assertEqual(result.blocks, params['tokens'])
+
+    def test_uris(self):
+        for uri in URIS:
+            text = self.wrap(uri)
+            hlinks = process_uris(text)
+            self.assertEqual(len(hlinks), 1, text)
+            self.assertEqual(hlinks[0].uri, uri, text)
+
+    def test_invalid_uris(self):
+        for foo in NONURIS + UNACCEPTABLE_URIS:
+            if foo == 'scheme:' or\
+               foo == 'http://':
+                continue  # these currently fial (FIXME)
+            text = self.wrap(foo)
+            hlinks = process_uris(text)
+            if len(hlinks) == 0:
+                continue
+            self.assertEqual(len(hlinks), 1, text)
+            self.assertNotEqual(hlinks[0].text, foo, text)
+
+    def test_jids(self):
+        for jid in JIDS:
+            text = self.wrap(jid)
+            hlinks = process_uris(text)
+            self.assertEqual(len(hlinks), 1, text)
+            self.assertEqual(hlinks[0].text, jid, text)
+            self.assertEqual(hlinks[0].uri, jid_to_iri(jid), text)
+
+    def test_nonjids(self):
+        for foo in NONJIDS:
+            if foo == 'juliet@':
+                continue  # this currently fials (FIXME)
+            text = self.wrap(foo)
+            hlinks = process_uris(text)
+            if len(hlinks) == 0:
+                continue
+            self.assertEqual(len(hlinks), 1, text)
+            self.assertNotEqual(hlinks[0].text, foo, text)
+
+    def test_uris_with_text(self):
+        for text, result in URIS_WITH_TEXT:
+            hlinks = process_uris(text)
+            self.assertEqual(len(hlinks), len(result), text)
             for i, res in enumerate(result):
-                self.assertTrue(match[i][0] == res)
-
-    def test_url_with_text(self):
-        for text, result in URL_WITH_TEXT:
-            match = URI_RX.search(text)
-            self.assertIsNotNone(match)
-            start = match.start()
-            end = match.end()
-            self.assertTrue(text[start:end] == result)
-
-    def test_xmpp_uris(self):
-        for text, result in XMPP_URIS:
-            match = ADDRESS_RX.search(text)
-            self.assertIsNotNone(match)
-            start = match.start()
-            end = match.end()
-            self.assertTrue(text[start:end] == result)
+                self.assertEqual(hlinks[i].text, res, text)
if __name__ == "__main__":
=====================================
test/no_gui/test_text_helpers.py
=====================================
@@ -0,0 +1,35 @@
+import unittest
+
+from gajim.common.text_helpers import escape_iri_path_segment
+from gajim.common.text_helpers import jid_to_iri
+
+
+class Test(unittest.TestCase):
+    def test_escape_iri_path_segment(self):
+        self.assertEqual(escape_iri_path_segment(''), '', '<empty string>')
+
+        über = 'u\u0308ber'
+        self.assertEqual(escape_iri_path_segment(über), über)
+
+        self.assertEqual(
+            escape_iri_path_segment(''.join(chr(c) for c in range(0x20,0x7F))),
+            '%20!%22%23$%25&'()*+,-.%2F0123456789:;%3C=%3E%3F@ABCDEFGHIJKLMN'
+            'OPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~',
+            'ASCII printable')
+
+        self.assertEqual(escape_iri_path_segment(
+            ''.join(chr(c) for c in range(0x01,0x20)) + chr(0x7F)),
+            ''.join('%%%02X'%c for c in range(0x01,0x20)) + '%7F',
+            'ASCII control (no null)')
+
+
+    def test_jid_to_iri(self):
+        jid = r'foo@bar'
+        self.assertEqual(jid_to_iri(jid), fr'xmpp:{jid}', jid)
+        jid = r'my\20self@[::1]/home'
+        self.assertEqual(jid_to_iri(jid),
+            r'xmpp:my%5C20self@%5B::1%5D/home', jid)
+
+
+if __name__ == '__main__':
+    unittest.main()
View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/1ef1e8fecf4517d3234ffa98b3e0a9245...
-- 
View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/1ef1e8fecf4517d3234ffa98b3e0a9245...
You're receiving this email because of your account on dev.gajim.org.





    

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

[Git][gajim/gajim][master] fix: Styling: Various link parsing issues