Philipp Hörist pushed to branch master at gajim / gajim
Commits: 851a7ee9 by Philipp Hörist at 2019-03-11T22:13:00Z Depend on nbxmpp delay parsing
- - - - - f5fe4fd8 by Philipp Hörist at 2019-03-11T22:13:11Z Better dedupe MUC history messages
- - - - - da46bdda by Philipp Hörist at 2019-03-11T22:13:11Z Remove delay parsing from Gajim
Use nbxmpp's delay parsing
- - - - - 348a8551 by Philipp Hörist at 2019-03-11T22:13:11Z Remove datetime parsing from Gajim
Use nbxmpp's datetime parsing
- - - - -
9 changed files:
- gajim/common/logger.py - gajim/common/modules/chatstates.py - − gajim/common/modules/date_and_time.py - gajim/common/modules/entity_time.py - gajim/common/modules/mam.py - gajim/common/modules/message.py - gajim/common/modules/misc.py - − test/no_gui/unit/test_date_time.py - − test/no_gui/unit/test_helpers.py
Changes:
===================================== gajim/common/logger.py ===================================== @@ -1234,6 +1234,53 @@ class Logger: self._con.executescript(sql) self._timeout_commit()
+ def deduplicate_muc_message(self, account, jid, resource, + timestamp, message_id): + """ + Check if a message is already in the `logs` table + + :param account: The account + + :param jid: The muc jid as string + + :param resource: The resource + + :param timestamp: The timestamp in UTC epoch + + :param message_id: The message-id + """ + + # Add 60 seconds around the timestamp + start_time = timestamp - 60 + end_time = timestamp + 60 + + account_id = self.get_account_id(account) + log.debug('Search for MUC duplicate') + log.debug('start: %s, end: %s, jid: %s, resource: %s, message-id: %s', + start_time, end_time, jid, resource, message_id) + + sql = ''' + SELECT * FROM logs + NATURAL JOIN jids WHERE + jid = ? AND + contact_name = ? AND + message_id = ? AND + account_id = ? AND + time BETWEEN ? AND ? + ''' + + result = self._con.execute(sql, (jid, + resource, + message_id, + account_id, + start_time, + end_time)).fetchone() + + if result is not None: + log.debug('Found duplicate') + return True + return False + def search_for_duplicate(self, account, jid, timestamp, msg): """ Check if a message is already in the `logs` table
===================================== gajim/common/modules/chatstates.py ===================================== @@ -30,7 +30,6 @@ from gi.repository import GLib from gajim.common import app from gajim.common.nec import NetworkEvent from gajim.common.const import Chatstate as State -from gajim.common.modules.misc import parse_delay from gajim.common.modules.base import BaseModule from gajim.common.connection_handlers_events import MessageOutgoingEvent from gajim.common.connection_handlers_events import GcMessageOutgoingEvent @@ -53,7 +52,7 @@ def ensure_enabled(func):
def parse_chatstate(stanza: nbxmpp.Message) -> Optional[str]: - if parse_delay(stanza) is not None: + if stanza.getTag('delay', namespace=nbxmpp.NS_DELAY2) is not None: return None
children = stanza.getChildren()
===================================== gajim/common/modules/date_and_time.py deleted ===================================== @@ -1,194 +0,0 @@ -# This file is part of Gajim. -# -# Gajim is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published -# by the Free Software Foundation; version 3 only. -# -# Gajim is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Gajim. If not, see http://www.gnu.org/licenses/. - -# XEP-0082: XMPP Date and Time Profiles - -import re -import time -import logging -from datetime import datetime -from datetime import timedelta -from datetime import timezone -from datetime import tzinfo - -log = logging.getLogger('gajim.c.m.date_and_time') - -PATTERN_DATETIME = re.compile( - r'([0-9]{4}-[0-9]{2}-[0-9]{2})' - r'T' - r'([0-9]{2}:[0-9]{2}:[0-9]{2})' - r'(.[0-9]{0,6})?' - r'(?:[0-9]+)?' - r'(?:(Z)|(?:([-+][0-9]{2}):([0-9]{2})))$' -) - -PATTERN_DELAY = re.compile( - r'([0-9]{4}-[0-9]{2}-[0-9]{2})' - r'T' - r'([0-9]{2}:[0-9]{2}:[0-9]{2})' - r'(.[0-9]{0,6})?' - r'(?:[0-9]+)?' - r'(?:(Z)|(?:([-+][0]{2}):([0]{2})))$' -) - - -ZERO = timedelta(0) -HOUR = timedelta(hours=1) -SECOND = timedelta(seconds=1) - -STDOFFSET = timedelta(seconds=-time.timezone) -if time.daylight: - DSTOFFSET = timedelta(seconds=-time.altzone) -else: - DSTOFFSET = STDOFFSET - -DSTDIFF = DSTOFFSET - STDOFFSET - - -class LocalTimezone(tzinfo): - ''' - A class capturing the platform's idea of local time. - May result in wrong values on historical times in - timezones where UTC offset and/or the DST rules had - changed in the past. - ''' - def fromutc(self, dt): - assert dt.tzinfo is self - stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND - args = time.localtime(stamp)[:6] - dst_diff = DSTDIFF // SECOND - # Detect fold - fold = (args == time.localtime(stamp - dst_diff)) - return datetime(*args, microsecond=dt.microsecond, - tzinfo=self, fold=fold) - - def utcoffset(self, dt): - if self._isdst(dt): - return DSTOFFSET - return STDOFFSET - - def dst(self, dt): - if self._isdst(dt): - return DSTDIFF - return ZERO - - def tzname(self, dt): - return 'local' - - @staticmethod - def _isdst(dt): - tt = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, - dt.weekday(), 0, 0) - stamp = time.mktime(tt) - tt = time.localtime(stamp) - return tt.tm_isdst > 0 - - -def create_tzinfo(hours=0, minutes=0, tz_string=None): - if tz_string is None: - return timezone(timedelta(hours=hours, minutes=minutes)) - - if tz_string.lower() == 'z': - return timezone.utc - - try: - hours, minutes = map(int, tz_string.split(':')) - except Exception: - log.warning('Wrong tz string: %s', tz_string) - return - - if hours not in range(-24, 24): - log.warning('Wrong tz string: %s', tz_string) - return - - if minutes not in range(0, 59): - log.warning('Wrong tz string: %s', tz_string) - return - - if hours in (24, -24) and minutes != 0: - log.warning('Wrong tz string: %s', tz_string) - return - return timezone(timedelta(hours=hours, minutes=minutes)) - - -def parse_datetime(timestring, check_utc=False, - convert='utc', epoch=False): - ''' - Parse a XEP-0082 DateTime Profile String - - :param timestring: a XEP-0082 DateTime profile formated string - - :param check_utc: if True, returns None if timestring is not - a timestring expressing UTC - - :param convert: convert the given timestring to utc or local time - - :param epoch: if True, returns the time in epoch - - Examples: - '2017-11-05T01:41:20Z' - '2017-11-05T01:41:20.123Z' - '2017-11-05T01:41:20.123+05:00' - - return a datetime or epoch - ''' - if convert not in (None, 'utc', 'local'): - raise TypeError('"%s" is not a valid value for convert') - if check_utc: - match = PATTERN_DELAY.match(timestring) - else: - match = PATTERN_DATETIME.match(timestring) - - if match: - timestring = ''.join(match.groups('')) - strformat = '%Y-%m-%d%H:%M:%S%z' - if match.group(3): - # Fractional second addendum to Time - strformat = '%Y-%m-%d%H:%M:%S.%f%z' - if match.group(4): - # UTC string denoted by addition of the character 'Z' - timestring = timestring[:-1] + '+0000' - try: - date_time = datetime.strptime(timestring, strformat) - except ValueError: - pass - else: - if check_utc: - if convert != 'utc': - raise ValueError( - 'check_utc can only be used with convert="utc"') - date_time.replace(tzinfo=timezone.utc) - if epoch: - return date_time.timestamp() - return date_time - - if convert == 'utc': - date_time = date_time.astimezone(timezone.utc) - if epoch: - return date_time.timestamp() - return date_time - - if epoch: - # epoch is always UTC, use convert='utc' or check_utc=True - raise ValueError( - 'epoch not available while converting to local') - - if convert == 'local': - date_time = date_time.astimezone(LocalTimezone()) - return date_time - - # convert=None - return date_time - return None
===================================== gajim/common/modules/entity_time.py ===================================== @@ -18,12 +18,12 @@ import time
import nbxmpp from nbxmpp.structs import StanzaHandler +from nbxmpp.modules.date_and_time import parse_datetime +from nbxmpp.modules.date_and_time import create_tzinfo
from gajim.common import app from gajim.common.nec import NetworkEvent from gajim.common.modules.base import BaseModule -from gajim.common.modules.date_and_time import parse_datetime -from gajim.common.modules.date_and_time import create_tzinfo
class EntityTime(BaseModule):
===================================== gajim/common/modules/mam.py ===================================== @@ -29,7 +29,6 @@ from gajim.common.const import SyncThreshold from gajim.common.caps_cache import muc_caps_cache from gajim.common.helpers import get_sync_threshold from gajim.common.helpers import AdditionalDataDict -from gajim.common.modules.misc import parse_delay from gajim.common.modules.misc import parse_oob from gajim.common.modules.misc import parse_correction from gajim.common.modules.util import get_eme_message @@ -138,9 +137,15 @@ class MAM(BaseModule): 'message-id: %s', stanza_id, message_id) raise nbxmpp.NodeProcessed
+ additional_data = AdditionalDataDict() + if properties.has_user_delay: + # Record it as a user timestamp + additional_data.set_value( + 'gajim', 'user_timestamp', properties.user_timestamp) + event_attrs.update( {'conn': self._con, - 'additional_data': AdditionalDataDict(), + 'additional_data': additional_data, 'encrypted': False, 'timestamp': properties.mam.timestamp, 'self_message': properties.is_self_message, @@ -205,12 +210,6 @@ class MAM(BaseModule): self._log.debug(event.message.getProperties()) return
- user_timestamp = parse_delay(event.stanza) - if user_timestamp is not None: - # Record it as a user timestamp - event.additional_data.set_value( - 'gajim', 'user_timestamp', user_timestamp) - event.correct_id = parse_correction(event.message) parse_oob(event)
===================================== gajim/common/modules/message.py ===================================== @@ -31,7 +31,6 @@ from gajim.common.modules.base import BaseModule from gajim.common.modules.util import get_eme_message from gajim.common.modules.security_labels import parse_securitylabel from gajim.common.modules.user_nickname import parse_nickname -from gajim.common.modules.misc import parse_delay from gajim.common.modules.misc import parse_correction from gajim.common.modules.misc import parse_attention from gajim.common.modules.misc import parse_form @@ -86,18 +85,20 @@ class Message(BaseModule):
type_ = properties.type
- # Check for duplicates stanza_id, message_id = self._get_unique_id(properties)
- # Check groupchat messages for duplicates, - # We do this because of MUC History messages - if (properties.type.is_groupchat or - properties.is_self_message or - properties.is_muc_pm): - if properties.type.is_groupchat: - archive_jid = stanza.getFrom().getStripped() - else: - archive_jid = self._con.get_own_jid().getStripped() + if properties.type.is_groupchat and properties.has_server_delay: + # Only for XEP-0045 MUC History + # Dont check for message text because the message could be encrypted + if app.logger.deduplicate_muc_message(self._account, + properties.jid.getBare(), + properties.jid.getResource(), + properties.timestamp, + properties.id): + raise nbxmpp.NodeProcessed + + if (properties.is_self_message or properties.is_muc_pm): + archive_jid = self._con.get_own_jid().getStripped() if app.logger.find_stanza_id(self._account, archive_jid, stanza_id, @@ -144,13 +145,19 @@ class Message(BaseModule):
session.last_receive = time.time()
+ additional_data = AdditionalDataDict() + + if properties.has_user_delay: + additional_data.set_value( + 'gajim', 'user_timestamp', properties.user_timestamp) + event_attr = { 'conn': self._con, 'stanza': stanza, 'account': self._account, 'id_': properties.id, 'encrypted': False, - 'additional_data': AdditionalDataDict(), + 'additional_data': additional_data, 'forwarded': forwarded, 'sent': sent, 'fjid': fjid, @@ -164,6 +171,8 @@ class Message(BaseModule): 'thread_id': thread_id, 'session': session, 'self_message': properties.is_self_message, + 'timestamp': properties.timestamp, + 'delayed': properties.user_timestamp is not None, 'muc_pm': properties.is_muc_pm, 'gc_control': gc_control } @@ -197,22 +206,6 @@ class Message(BaseModule): subject = event.stanza.getSubject() groupchat = event.mtype == 'groupchat'
- # Determine timestamps - if groupchat: - delay_jid = event.jid - else: - delay_jid = self._con.get_own_jid().getDomain() - timestamp = parse_delay(event.stanza, from_=delay_jid) - if timestamp is None: - timestamp = time.time() - - user_timestamp = parse_delay(event.stanza, - not_from=[delay_jid]) - - if user_timestamp is not None: - event.additional_data.set_value( - 'gajim', 'user_timestamp', user_timestamp) - event_attr = { 'popup': False, 'msg_log_id': None, @@ -223,8 +216,6 @@ class Message(BaseModule): 'user_nick': '' if event.sent else parse_nickname(event.stanza), 'form_node': parse_form(event.stanza), 'xhtml': parse_xhtml(event.stanza), - 'timestamp': timestamp, - 'delayed': user_timestamp is not None, } parse_oob(event)
===================================== gajim/common/modules/misc.py ===================================== @@ -19,54 +19,10 @@ import logging import nbxmpp
from gajim.common import app -from gajim.common.modules.date_and_time import parse_datetime
log = logging.getLogger('gajim.c.m.misc')
-# XEP-0203: Delayed Delivery - -def parse_delay(stanza, epoch=True, convert='utc', from_=None, not_from=None): - ''' - Returns the first valid delay timestamp that matches - - :param epoch: Returns the timestamp as epoch - - :param convert: Converts the timestamp to either utc or local - - :param from_: Matches only delays that have the according - from attr set - - :param not_from: Matches only delays that have the according - from attr not set - ''' - delays = stanza.getTags('delay', namespace=nbxmpp.NS_DELAY2) - - for delay in delays: - stamp = delay.getAttr('stamp') - if stamp is None: - log.warning('Invalid timestamp received: %s', stamp) - log.warning(stanza) - continue - - delay_from = delay.getAttr('from') - if from_ is not None: - if delay_from != from_: - continue - if not_from is not None: - if delay_from in not_from: - continue - - timestamp = parse_datetime(stamp, check_utc=True, - epoch=epoch, convert=convert) - if timestamp is None: - log.warning('Invalid timestamp received: %s', stamp) - log.warning(stanza) - continue - - return timestamp - - # XEP-0066: Out of Band Data
def parse_oob(event):
===================================== test/no_gui/unit/test_date_time.py deleted ===================================== @@ -1,99 +0,0 @@ -import unittest -from datetime import datetime -from datetime import timezone -from datetime import timedelta - -from gajim.common.modules.date_and_time import parse_datetime -from gajim.common.modules.date_and_time import LocalTimezone -from gajim.common.modules.date_and_time import create_tzinfo - - -class TestDateTime(unittest.TestCase): - - def test_convert_to_utc(self): - - strings = { - # Valid UTC strings and fractions - '2017-11-05T01:41:20Z': 1509846080.0, - '2017-11-05T01:41:20.123Z': 1509846080.123, - '2017-11-05T01:41:20.123123123+00:00': 1509846080.123123, - '2017-11-05T01:41:20.123123123123123-00:00': 1509846080.123123, - - # Invalid strings - '2017-11-05T01:41:20Z+05:00': None, - '2017-11-05T01:41:20+0000': None, - '2017-11-05T01:41:20-0000': None, - - # Valid strings with offset - '2017-11-05T01:41:20-05:00': 1509864080.0, - '2017-11-05T01:41:20+05:00': 1509828080.0, - } - - strings2 = { - # Valid strings with offset - '2017-11-05T01:41:20-05:00': datetime(2017, 11, 5, 1, 41, 20, 0, create_tzinfo(hours=-5)), - '2017-11-05T01:41:20+05:00': datetime(2017, 11, 5, 1, 41, 20, 0, create_tzinfo(hours=5)), - } - - for time_string, expected_value in strings.items(): - result = parse_datetime(time_string, convert='utc', epoch=True) - self.assertEqual(result, expected_value) - - for time_string, expected_value in strings2.items(): - result = parse_datetime(time_string, convert='utc') - self.assertEqual(result, expected_value.astimezone(timezone.utc)) - - def test_convert_to_local(self): - - strings = { - # Valid UTC strings and fractions - '2017-11-05T01:41:20Z': datetime(2017, 11, 5, 1, 41, 20, 0, timezone.utc), - '2017-11-05T01:41:20.123Z': datetime(2017, 11, 5, 1, 41, 20, 123000, timezone.utc), - '2017-11-05T01:41:20.123123123+00:00': datetime(2017, 11, 5, 1, 41, 20, 123123, timezone.utc), - '2017-11-05T01:41:20.123123123123123-00:00': datetime(2017, 11, 5, 1, 41, 20, 123123, timezone.utc), - - # Valid strings with offset - '2017-11-05T01:41:20-05:00': datetime(2017, 11, 5, 1, 41, 20, 0, create_tzinfo(hours=-5)), - '2017-11-05T01:41:20+05:00': datetime(2017, 11, 5, 1, 41, 20, 0, create_tzinfo(hours=5)), - } - - for time_string, expected_value in strings.items(): - result = parse_datetime(time_string, convert='local') - self.assertEqual(result, expected_value.astimezone(LocalTimezone())) - - def test_no_convert(self): - - strings = { - # Valid UTC strings and fractions - '2017-11-05T01:41:20Z': timedelta(0), - '2017-11-05T01:41:20.123Z': timedelta(0), - '2017-11-05T01:41:20.123123123+00:00': timedelta(0), - '2017-11-05T01:41:20.123123123123123-00:00': timedelta(0), - - # Valid strings with offset - '2017-11-05T01:41:20-05:00': timedelta(hours=-5), - '2017-11-05T01:41:20+05:00': timedelta(hours=5), - } - - for time_string, expected_value in strings.items(): - result = parse_datetime(time_string, convert=None) - self.assertEqual(result.utcoffset(), expected_value) - - def test_check_utc(self): - - strings = { - # Valid UTC strings and fractions - '2017-11-05T01:41:20Z': 1509846080.0, - '2017-11-05T01:41:20.123Z': 1509846080.123, - '2017-11-05T01:41:20.123123123+00:00': 1509846080.123123, - '2017-11-05T01:41:20.123123123123123-00:00': 1509846080.123123, - - # Valid strings with offset - '2017-11-05T01:41:20-05:00': None, - '2017-11-05T01:41:20+05:00': None, - } - - for time_string, expected_value in strings.items(): - result = parse_datetime( - time_string, check_utc=True, epoch=True) - self.assertEqual(result, expected_value)
===================================== test/no_gui/unit/test_helpers.py deleted ===================================== @@ -1,30 +0,0 @@ -import unittest - -import nbxmpp - -from gajim.common.modules.misc import parse_delay - -class TestHelpers(unittest.TestCase): - - def test_parse_delay(self): - - node = """ - <message> - <delay xmlns='urn:xmpp:delay' from='capulet.com' stamp='2002-09-10T23:08:25Z' /> - <delay xmlns='urn:xmpp:delay' from='romeo.com' stamp='2010-09-10T23:08:25Z' /> - <delay xmlns='urn:xmpp:delay' stamp='2015-09-10T23:08:25Z' /> - </message> - """ - message = nbxmpp.Node(node=node) - - timestamp = parse_delay(message) - self.assertEqual(timestamp, 1031699305.0) - - timestamp = parse_delay(message, from_='capulet.com') - self.assertEqual(timestamp, 1031699305.0) - - timestamp = parse_delay(message, from_='romeo.com') - self.assertEqual(timestamp, 1284160105.0) - - timestamp = parse_delay(message, not_from=['romeo.com']) - self.assertEqual(timestamp, 1031699305.0) \ No newline at end of file
View it on GitLab: https://dev.gajim.org/gajim/gajim/compare/b80bbaf2b31cc60a5677c42145f07a833b...