Commits August 2017

commits@gajim.org

1 participants
94 discussions

[Git][gajim/gajim][master] Stay compatible to python 3.4
by Philipp Hörist 02 Aug '17

02 Aug '17

Philipp Hörist pushed to branch master at gajim / gajim Commits: 6f3c2111 by Philipp Hörist at 2017-08-02T18:57:02+02:00 Stay compatible to python 3.4 Fixes #8690 - - - - - 2 changed files: - gajim/common/logger.py - gajim/emoticons.py Changes: ===================================== gajim/common/logger.py ===================================== --- a/gajim/common/logger.py +++ b/gajim/common/logger.py @@ -676,7 +676,7 @@ class Logger: try: messages = self.con.execute( - sql, (*jids, restore, pending)).fetchall() + sql, tuple(jids) + (restore, pending)).fetchall() except sqlite.DatabaseError: self.dispatch('DB_ERROR', exceptions.DatabaseMalformed(LOG_DB_PATH)) @@ -722,8 +722,8 @@ class Logger: ORDER BY time, log_line_id '''.format(jids=', '.join('?' * len(jids))) - return self.con.execute(sql, (*jids, - date.timestamp(), + return self.con.execute(sql, tuple(jids) + + (date.timestamp(), (date + delta).timestamp())).fetchall() def search_log(self, account, jid, query, date=None): @@ -765,7 +765,7 @@ class Logger: '''.format(jids=', '.join('?' * len(jids)), date_search=between if date else '') - return self.con.execute(sql, (*jids, query)).fetchall() + return self.con.execute(sql, tuple(jids) + (query,)).fetchall() def get_days_with_logs(self, account, jid, year, month): """ @@ -803,8 +803,8 @@ class Logger: """.format(jids=', '.join('?' * len(jids)), kinds=', '.join(kinds)) - return self.con.execute(sql, (*jids, - date.timestamp(), + return self.con.execute(sql, tuple(jids) + + (date.timestamp(), (date + delta).timestamp())).fetchall() def get_last_date_that_has_logs(self, account, jid): @@ -831,7 +831,7 @@ class Logger: # fetchone() returns always at least one Row with all # attributes set to None because of the MAX() function - return self.con.execute(sql, (*jids,)).fetchone().time + return self.con.execute(sql, tuple(jids)).fetchone().time def get_room_last_message_time(self, account, jid): """ @@ -1210,7 +1210,7 @@ class Logger: '''.format(columns=', '.join(kwargs.keys()), values=', '.join('?' * len(kwargs))) - lastrowid = self.con.execute(sql, (jid, time_, kind, *kwargs.values())).lastrowid + lastrowid = self.con.execute(sql, (jid, time_, kind) + tuple(kwargs.values())).lastrowid if unread and kind == KindConstant.CHAT_MSG_RECV: sql = '''INSERT INTO unread_messages (message_id, jid_id) ===================================== gajim/emoticons.py ===================================== --- a/gajim/emoticons.py +++ b/gajim/emoticons.py @@ -20,6 +20,7 @@ import sys import logging import importlib.util as imp from collections import OrderedDict +from importlib.machinery import SourceFileLoader from gi.repository import GdkPixbuf, Gtk, GLib @@ -65,10 +66,9 @@ def load(path): module_name = 'emoticons_theme.pyc' theme_path = os.path.join(path, module_name) - spec = imp.spec_from_file_location(module_name, theme_path) + loader = SourceFileLoader(module_name, theme_path) try: - theme = imp.module_from_spec(spec) - spec.loader.exec_module(theme) + theme = loader.load_module() except FileNotFoundError: log.exception('Emoticons theme not found') return View it on GitLab: https://dev.gajim.org/gajim/gajim/commit/6f3c21118f9d1c88d1a725381dea963d70… --- View it on GitLab: https://dev.gajim.org/gajim/gajim/commit/6f3c21118f9d1c88d1a725381dea963d70… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] 5 commits: Add new logger methods
by Philipp Hörist 01 Aug '17

01 Aug '17

Philipp Hörist pushed to branch master at gajim / gajim Commits: 16fd6487 by Philipp Hörist at 2017-08-01T21:39:06+02:00 Add new logger methods Goal is to make the logger module easier to maintain in the future - extract deduplication out into a own method, so we can use it on its own in some other places. - add new insert_into_logs() method the DB fields we want to write to are passed as arguments, this makes it so we dont have to change the method when we add or remove DB fields in the future. - instead of using get_jid_id() all around gajim to get the jid_id before we write to the DB, we only have to pass the jid to insert_into_logs(), it makes sure to get the correct jid_id or add one in case it doesnt exist. - - - - - 1f5e927c by Philipp Hörist at 2017-08-01T21:39:11+02:00 Use new DB methods for MAM Messages - also push a new MamDecryptedMessageReceived event after disco instead of writing directly to the DB - - - - - e24d4f8c by Philipp Hörist at 2017-08-01T21:39:11+02:00 Fix receiving MAM Messages from ourself This is a regression from the refactoring - - - - - 651611b2 by Philipp Hörist at 2017-08-01T21:43:28+02:00 Improve timestamp usage for MAM - Use new parse_datetime() method - Drop message with error if MAM doesnt supply a timestamp. - If the user supplys an own timestamp, save it so we can decide in the future how to display it. - - - - - bc7ce129 by Philipp Hörist at 2017-08-01T23:09:22+02:00 Merge branch 'logger' into 'master' New logging methods and more MAM refactoring See merge request !112 - - - - - 4 changed files: - gajim/common/connection_handlers_events.py - gajim/common/helpers.py - gajim/common/logger.py - gajim/common/message_archiving.py Changes: ===================================== gajim/common/connection_handlers_events.py ===================================== --- a/gajim/common/connection_handlers_events.py +++ b/gajim/common/connection_handlers_events.py @@ -40,7 +40,7 @@ from common import i18n from common import dataforms from common import exceptions from common.zeroconf.zeroconf import Constant -from common.logger import LOG_DB_PATH +from common.logger import LOG_DB_PATH, KindConstant from common.pep import SUPPORTED_PERSONAL_USER_EVENTS from common.jingle_transport import JingleTransportSocks5 from common.file_props import FilesProp @@ -1051,15 +1051,19 @@ class MamMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent): self.query_id = self.result.getAttr('queryid') frm = self.msg_.getFrom() + # Some servers dont set the 'to' attribute when + # we send a message to ourself to = self.msg_.getTo() + if to is None: + to = own_jid if frm.bareMatch(own_jid): self.stanza_id = self.msg_.getOriginID() - if not self.stanza_id: + if self.stanza_id is None: self.stanza_id = self.msg_.getID() self.with_ = to - self.direction = 'to' + self.kind = KindConstant.CHAT_MSG_SENT else: if self.result.getNamespace() == nbxmpp.NS_MAM_2: self.stanza_id = self.result.getID() @@ -1067,22 +1071,32 @@ class MamMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent): self.stanza_id = self.msg_.getID() self.with_ = frm - self.direction = 'from' + self.kind = KindConstant.CHAT_MSG_RECV - if not self.stanza_id: + if self.stanza_id is None: log.debug('Could not retrieve stanza-id') - # Use timestamp provided by archive, - # Fallback: Use timestamp provided by user and issue a warning - delay = self.forwarded.getTag('delay', namespace=nbxmpp.NS_DELAY2) - if not delay: - log.warning('No timestamp on archive Message, try fallback') - delay = self.msg_.getTag('delay', namespace=nbxmpp.NS_DELAY2) - if not delay: + delay = self.forwarded.getTagAttr( + 'delay', 'stamp', namespace=nbxmpp.NS_DELAY2) + if delay is None: log.error('Received MAM message without timestamp') return - self.timestamp = helpers.parse_delay(delay) + self.timestamp = helpers.parse_datetime( + delay, check_utc=True, epoch=True) + if self.timestamp is None: + log.error('Received MAM message with invalid timestamp: %s', delay) + return + + # Save timestamp added by the user + user_delay = self.msg_.getTagAttr( + 'delay', 'stamp', namespace=nbxmpp.NS_DELAY2) + if user_delay is not None: + self.user_timestamp = helpers.parse_datetime( + user_delay, check_utc=True, epoch=True) + if self.user_timestamp is None: + log.warning('Received MAM message with ' + 'invalid user timestamp: %s', user_delay) log.debug('Received mam-message: stanza id: %s', self.stanza_id) return True @@ -1095,19 +1109,27 @@ class MamDecryptedMessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent): if not self.msgtxt: # For example Chatstates, Receipts, Chatmarkers log.debug('Received MAM message without text') - return False + return self.is_pm = gajim.logger.jid_is_room_jid(self.with_.getStripped()) if self.is_pm is None: + # Check if this event is triggered after a disco, so we dont + # run into an endless loop + if hasattr(self, 'disco'): + log.error('JID not known even after sucessful disco') + return # we don't know this JID, we need to disco it. server = self.with_.getDomain() if server not in self.conn.mam_awaiting_disco_result: - self.conn.mam_awaiting_disco_result[server] = [ - [self.with_, self.direction, self.timestamp, self.msgtxt]] + self.conn.mam_awaiting_disco_result[server] = [self] self.conn.discoverInfo(server) else: - self.conn.mam_awaiting_disco_result[server].append( - [self.with_, self.direction, self.timestamp, self.msgtxt]) + self.conn.mam_awaiting_disco_result[server].append(self) return + + if self.is_pm: + self.with_ = str(self.with_) + else: + self.with_ = self.with_.getStripped() return True class MessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent): ===================================== gajim/common/helpers.py ===================================== --- a/gajim/common/helpers.py +++ b/gajim/common/helpers.py @@ -673,34 +673,6 @@ def datetime_tuple(timestamp): tim = tim.timetuple() return tim -def parse_delay(timestamp): - ''' - Parse a timestamp - https://xmpp.org/extensions/xep-0203.html - Note: Not all delay tags should be parsed with this method - see https://xmpp.org/extensions/xep-0082.html for more information - - :param timestamp: a XEP-0203 fomated timestring string or a delay Node - - Examples: - '2017-11-05T01:41:20Z' - '2017-11-05T01:41:20.123Z' - - return epoch UTC timestamp - ''' - if isinstance(timestamp, nbxmpp.protocol.Node): - timestamp = timestamp.getAttr('stamp') - timestamp += '+0000' - try: - datetime_ = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ%z') - except ValueError: - try: - datetime_ = datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ%z') - except ValueError: - log.error('Could not parse delay timestamp: %s', timestamp) - raise - return datetime_.timestamp() - def parse_datetime(timestring, check_utc=False, convert='utc', epoch=False): ''' Parse a XEP-0082 DateTime Profile String ===================================== gajim/common/logger.py ===================================== --- a/gajim/common/logger.py +++ b/gajim/common/logger.py @@ -1139,25 +1139,23 @@ class Logger: (account_jid_id,)) self._timeout_commit() - def save_if_not_exists(self, with_, direction, tim, msg, is_pm=False, additional_data=None): - if additional_data is None: - additional_data = {} + def search_for_duplicate(self, jid, timestamp, msg): + """ + Check if a message is already in the `logs` table - if is_pm: - with_ = str(with_) - type_ = 'gc_msg' - else: - with_ = with_.getStripped() - if direction == 'from': - type_ = 'chat_msg_recv' - elif direction == 'to': - type_ = 'chat_msg_sent' + :param jid: The jid as string - start_time = tim - 300 # 5 minutes arrount given time - end_time = tim + 300 # 5 minutes arrount given time + :param timestamp: The timestamp in UTC epoch + + :param msg: The message text + """ + + # Add 5 minutes around the timestamp + start_time = timestamp - 300 + end_time = timestamp + 300 log.debug('start: %s, end: %s, jid: %s, message: %s', - start_time, end_time, with_, msg) + start_time, end_time, jid, msg) sql = ''' SELECT * FROM logs @@ -1165,14 +1163,63 @@ class Logger: AND time BETWEEN ? AND ? ''' - result = self.con.execute(sql, (with_, msg, start_time, end_time)).fetchone() + result = self.con.execute(sql, (jid, msg, start_time, end_time)).fetchone() - if result: - log.debug('Log already in DB, ignoring it') - return - log.debug('New log received from server archives, storing it') - self.write(type_, with_, message=msg, tim=tim, - additional_data=additional_data, mam_query=True) + if result is not None: + log.debug('Message already in DB') + return True + return False + + def insert_jid(self, jid, kind=None, type_=JIDConstant.NORMAL_TYPE): + """ + Insert a new jid into the `jids` table. + + :param jid: The jid as string + + :param kind: A KindConstant + + :param type_: A JIDConstant + """ + if kind == KindConstant.GC_MSG: + type_ = JIDConstant.ROOM_TYPE + sql = 'INSERT OR IGNORE INTO jids (jid, type) VALUES (?, ?)' + self.con.execute(sql, (jid, type_)) + self._timeout_commit() + + def insert_into_logs(self, jid, time_, kind, unread=True, **kwargs): + """ + Insert a new message into the `logs` table + + :param jid: The jid as string + + :param time_: The timestamp in UTC epoch + + :param kind: A KindConstant + + :param unread: If True the message is added to the`unread_messages` + table. Only if kind == CHAT_MSG_RECV + + :param kwargs: Every additional named argument must correspond to + a field in the `logs` table + """ + self.insert_jid(jid, kind=kind) + + sql = ''' + INSERT INTO logs (jid_id, time, kind, {columns}) + VALUES ((SELECT jid_id FROM jids WHERE jid = ?), ?, ?, {values}) + '''.format(columns=', '.join(kwargs.keys()), + values=', '.join('?' * len(kwargs))) + + lastrowid = self.con.execute(sql, (jid, time_, kind, *kwargs.values())).lastrowid + + if unread and kind == KindConstant.CHAT_MSG_RECV: + sql = '''INSERT INTO unread_messages (message_id, jid_id) + VALUES (?, (SELECT jid_id FROM jids WHERE jid = ?))''' + self.con.execute(sql, (lastrowid, jid)) + + self._timeout_commit() + + return lastrowid def _nec_gc_message_received(self, obj): tim_f = float(obj.timestamp) ===================================== gajim/common/message_archiving.py ===================================== --- a/gajim/common/message_archiving.py +++ b/gajim/common/message_archiving.py @@ -25,6 +25,7 @@ import nbxmpp from common import gajim from common import ged +from common.logger import KindConstant, JIDConstant import common.connection_handlers_events as ev log = logging.getLogger('gajim.c.message_archiving') @@ -77,23 +78,28 @@ class ConnectionArchive313: del self.mam_awaiting_disco_result[obj.jid] def _nec_agent_info(self, obj): - if obj.jid in self.mam_awaiting_disco_result: - for identity in obj.identities: - if identity['category'] == 'conference': - # it's a groupchat - for with_, direction, tim, msg_txt in \ - self.mam_awaiting_disco_result[obj.jid]: - gajim.logger.get_jid_id(with_.getStripped(), 'ROOM') - gajim.logger.save_if_not_exists(with_, direction, tim, - msg_txt, is_pm=True) - del self.mam_awaiting_disco_result[obj.jid] - return - # it's not a groupchat - for with_, direction, tim, msg_txt in \ - self.mam_awaiting_disco_result[obj.jid]: - gajim.logger.get_jid_id(with_.getStripped()) - gajim.logger.save_if_not_exists(with_, direction, tim, msg_txt) + if obj.jid not in self.mam_awaiting_disco_result: + return + + for identity in obj.identities: + if identity['category'] != 'conference': + continue + # it's a groupchat + for msg_obj in self.mam_awaiting_disco_result[obj.jid]: + gajim.logger.insert_jid(msg_obj.with_.getStripped(), + type_=JIDConstant.ROOM_TYPE) + gajim.nec.push_incoming_event( + ev.MamDecryptedMessageReceivedEvent( + None, disco=True, **vars(msg_obj))) del self.mam_awaiting_disco_result[obj.jid] + return + # it's not a groupchat + for msg_obj in self.mam_awaiting_disco_result[obj.jid]: + gajim.logger.insert_jid(msg_obj.with_.getStripped()) + gajim.nec.push_incoming_event( + ev.MamDecryptedMessageReceivedEvent( + None, disco=True, **vars(msg_obj))) + del self.mam_awaiting_disco_result[obj.jid] def _nec_result_finished(self, obj): if obj.conn.name != self.name: @@ -121,8 +127,13 @@ class ConnectionArchive313: def _nec_mam_decrypted_message_received(self, obj): if obj.conn.name != self.name: return - gajim.logger.save_if_not_exists(obj.with_, obj.direction, obj.timestamp, - obj.msgtxt, is_pm=obj.is_pm, additional_data=obj.additional_data) + duplicate = gajim.logger.search_for_duplicate( + obj.with_, obj.timestamp, obj.msgtxt) + if not duplicate: + gajim.logger.insert_into_logs( + obj.with_, obj.timestamp, obj.kind, + unread=False, + message=obj.msgtxt) def get_query_id(self): self.mam_query_id = self.connection.getAnID() View it on GitLab: https://dev.gajim.org/gajim/gajim/compare/6925ce0628a6028c92562fd1d22ca4a7d… --- View it on GitLab: https://dev.gajim.org/gajim/gajim/compare/6925ce0628a6028c92562fd1d22ca4a7d… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] 2 commits: Add a new XEP-0082 parsing method
by Philipp Hörist 01 Aug '17

01 Aug '17

Philipp Hörist pushed to branch master at gajim / gajim Commits: 0fee2792 by Philipp Hörist at 2017-08-01T20:53:00+02:00 Add a new XEP-0082 parsing method This handles every possible XEP-0082 timestamp It has some additional options: - Check if a timestring is a valid UTC timestamp, as required by some XEPs (for example: XEP-0203) - Return timestamp as datetime in UTC - Return timestamp as datetime in localtime - Return timestamp as epoch - - - - - 6925ce06 by Philipp Hörist at 2017-08-01T20:57:15+02:00 Merge branch 'master' into 'master' Add a new XEP-0082 parsing method See merge request !111 - - - - - 1 changed file: - gajim/common/helpers.py Changes: ===================================== gajim/common/helpers.py ===================================== --- a/gajim/common/helpers.py +++ b/gajim/common/helpers.py @@ -43,7 +43,7 @@ import shlex from common import caps_cache import socket import time -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone, tzinfo from encodings.punycode import punycode_encode from string import Template @@ -81,6 +81,78 @@ log = logging.getLogger('gajim.c.helpers') special_groups = (_('Transports'), _('Not in Roster'), _('Observers'), _('Groupchats')) +# Patterns for DateTime parsing XEP-0082 +PATTERN_DATETIME = re.compile( + r'([0-9]{4}-[0-9]{2}-[0-9]{2})' + r'T' + r'([0-9]{2}:[0-9]{2}:[0-9]{2})' + r'(\.[0-9]{0,6})?' + r'(?:[0-9]+)?' + r'(?:(Z)|(?:([-+][0-9]{2}):([0-9]{2})))$' + ) + +PATTERN_DELAY = re.compile( + r'([0-9]{4}-[0-9]{2}-[0-9]{2})' + r'T' + r'([0-9]{2}:[0-9]{2}:[0-9]{2})' + r'(\.[0-9]{0,6})?' + r'(?:[0-9]+)?' + r'(?:(Z)|(?:([-+][0]{2}):([0]{2})))$' + ) + +ZERO = timedelta(0) +HOUR = timedelta(hours=1) +SECOND = timedelta(seconds=1) + +STDOFFSET = timedelta(seconds=-time.timezone) +if time.daylight: + DSTOFFSET = timedelta(seconds=-time.altzone) +else: + DSTOFFSET = STDOFFSET + +DSTDIFF = DSTOFFSET - STDOFFSET + + +class LocalTimezone(tzinfo): + ''' + A class capturing the platform's idea of local time. + May result in wrong values on historical times in + timezones where UTC offset and/or the DST rules had + changed in the past. + ''' + def fromutc(self, dt): + assert dt.tzinfo is self + stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND + args = time.localtime(stamp)[:6] + dst_diff = DSTDIFF // SECOND + # Detect fold + fold = (args == time.localtime(stamp - dst_diff)) + return datetime(*args, microsecond=dt.microsecond, + tzinfo=self, fold=fold) + + def utcoffset(self, dt): + if self._isdst(dt): + return DSTOFFSET + else: + return STDOFFSET + + def dst(self, dt): + if self._isdst(dt): + return DSTDIFF + else: + return ZERO + + def tzname(self, dt): + return 'local' + + def _isdst(self, dt): + tt = (dt.year, dt.month, dt.day, + dt.hour, dt.minute, dt.second, + dt.weekday(), 0, 0) + stamp = time.mktime(tt) + tt = time.localtime(stamp) + return tt.tm_isdst > 0 + class InvalidFormat(Exception): pass @@ -629,6 +701,56 @@ def parse_delay(timestamp): raise return datetime_.timestamp() +def parse_datetime(timestring, check_utc=False, convert='utc', epoch=False): + ''' + Parse a XEP-0082 DateTime Profile String + https://xmpp.org/extensions/xep-0082.html + + :param timestring: a XEP-0082 DateTime profile formated string + + :param check_utc: if True, returns None if timestring is not + a timestring expressing UTC + + :param convert: convert the given timestring to utc or local time + + :param epoch: if True, returns the time in epoch + + Examples: + '2017-11-05T01:41:20Z' + '2017-11-05T01:41:20.123Z' + '2017-11-05T01:41:20.123+05:00' + + return a datetime or epoch + ''' + if convert not in (None, 'utc', 'local'): + raise TypeError('"%s" is not a valid value for convert') + if check_utc: + match = PATTERN_DELAY.match(timestring) + else: + match = PATTERN_DATETIME.match(timestring) + + if match: + timestring = ''.join(match.groups('')) + strformat = '%Y-%m-%d%H:%M:%S%z' + if match.group(3): + # Fractional second addendum to Time + strformat = '%Y-%m-%d%H:%M:%S.%f%z' + if match.group(4): + # UTC string denoted by addition of the character 'Z' + timestring = timestring[:-1] + '+0000' + try: + date_time = datetime.strptime(timestring, strformat) + except ValueError: + pass + else: + if not check_utc and convert == 'utc': + date_time = date_time.astimezone(timezone.utc) + if convert == 'local': + date_time = date_time.astimezone(LocalTimezone()) + if epoch: + return date_time.timestamp() + return date_time + return None from common import gajim if gajim.HAVE_PYCURL: View it on GitLab: https://dev.gajim.org/gajim/gajim/compare/63112d394e8f4f4f26875a2b1b267b019… --- View it on GitLab: https://dev.gajim.org/gajim/gajim/compare/63112d394e8f4f4f26875a2b1b267b019… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] Add Catalan translation file
by Yann Leboulanger 01 Aug '17

01 Aug '17

Yann Leboulanger pushed to branch master at gajim / gajim Commits: 63112d39 by Yann Leboulanger at 2017-08-01T14:05:53+02:00 Add Catalan translation file - - - - - 1 changed file: - + po/ca.po Changes: ===================================== po/ca.po ===================================== The diff for this file was not included because it is too large. View it on GitLab: https://dev.gajim.org/gajim/gajim/commit/63112d394e8f4f4f26875a2b1b267b0196… --- View it on GitLab: https://dev.gajim.org/gajim/gajim/commit/63112d394e8f4f4f26875a2b1b267b0196… You're receiving this email because of your account on dev.gajim.org.

1 0

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

Commits August 2017