Philipp Hörist pushed to branch master at gajim / gajim

Commits: 608655de by Philipp Hörist at 2017-08-08T14:41:55+02:00 Use new DB API for logging status and errors

- - - - - 8c920b35 by Philipp Hörist at 2017-08-08T15:24:41+02:00 Move logging GC messages into connection_handlers

- It should be where all other message received handlers are - Port to new DB API

- - - - - 25a764a6 by Philipp Hörist at 2017-08-08T18:22:23+02:00 Port logging incoming messages to new DB API

- - - - - f88e8d83 by Philipp Hörist at 2017-08-08T18:22:29+02:00 Remove old DB methods

- - - - - c53e2b1b by Philipp Hörist at 2017-08-08T18:22:29+02:00 Dont try to insert if JID is already in DB

- This is added because even an ignored INSERT raises the autoincrement value of the table. This means the jid id gets high really fast.

- - - - - 10b08652 by Philipp Hörist at 2017-08-08T18:22:29+02:00 Refactor set_room_last_message_time()

- Dont depend on get_jid_id(), this saves roundtrips

- - - - - f3bdb596 by Philipp Hörist at 2017-08-08T18:53:25+02:00 Refactor remove_roster()

- Execute as script to save roundtrips

- - - - -

4 changed files:

- gajim/common/connection_handlers.py - gajim/common/connection_handlers_events.py - gajim/common/logger.py - gajim/session.py

Changes:

===================================== gajim/common/connection_handlers.py ===================================== --- a/gajim/common/connection_handlers.py +++ b/gajim/common/connection_handlers.py @@ -897,6 +897,8 @@ class ConnectionHandlersBase: self._nec_message_received) gajim.ged.register_event_handler('decrypted-message-received', ged.CORE, self._nec_decrypted_message_received) + gajim.ged.register_event_handler('gc-message-received', ged.CORE, + self._nec_gc_message_received)

def cleanup(self): gajim.ged.remove_event_handler('iq-error-received', ged.CORE, @@ -911,6 +913,8 @@ class ConnectionHandlersBase: self._nec_message_received) gajim.ged.remove_event_handler('decrypted-message-received', ged.CORE, self._nec_decrypted_message_received) + gajim.ged.remove_event_handler('gc-message-received', ged.CORE, + self._nec_gc_message_received)

def _nec_iq_error_received(self, obj): if obj.conn.name != self.name: @@ -1059,7 +1063,13 @@ class ConnectionHandlersBase:

if gajim.config.get('log_contact_status_changes') and \ gajim.config.should_log(self.name, obj.jid): - gajim.logger.write('status', obj.jid, obj.status, obj.show) + show = gajim.logger.convert_show_values_to_db_api_values(obj.show) + if show is not None: + gajim.logger.insert_into_logs(nbxmpp.JID(obj.jid).getStripped(), + time_time(), + KindConstant.STATUS, + message=obj.status, + show=show)

def _nec_gc_presence_received(self, obj): if obj.conn.name != self.name: @@ -1151,6 +1161,24 @@ class ConnectionHandlersBase: conn=self, msg_obj=obj)) return True

+ def _nec_gc_message_received(self, obj): + if gajim.config.should_log(obj.conn.name, obj.jid) and not \ + obj.timestamp < obj.conn.last_history_time[obj.jid] and obj.msgtxt and \ + obj.nick: + # if not obj.nick, it means message comes from room itself + # usually it hold description and can be send at each connection + # so don't store it in logs + gajim.logger.insert_into_logs(obj.jid, + obj.timestamp, + KindConstant.GC_MSG, + message=obj.msgtxt, + contact_name=obj.nick, + additional_data=obj.additional_data) + # store in memory time of last message logged. + # this will also be saved in rooms_last_message_time table + # when we quit this muc + obj.conn.last_history_time[obj.jid] = obj.timestamp + # process and dispatch an error message def dispatch_error_message(self, msg, msgtxt, session, frm, tim): error_msg = msg.getErrorMsg() @@ -1162,8 +1190,12 @@ class ConnectionHandlersBase: subject = msg.getSubject()

if session.is_loggable(): - gajim.logger.write('error', frm, error_msg, tim=tim, - subject=subject) + gajim.logger.insert_into_logs(nbxmpp.JID(frm).getStripped(), + tim, + KindConstant.ERROR, + message=error_msg, + subject=subject) + gajim.nec.push_incoming_event(MessageErrorEvent(None, conn=self, fjid=frm, error_code=msg.getErrorCode(), error_msg=error_msg, msg=msgtxt, time_=tim, session=session, stanza=msg))

===================================== gajim/common/connection_handlers_events.py ===================================== --- a/gajim/common/connection_handlers_events.py +++ b/gajim/common/connection_handlers_events.py @@ -932,7 +932,15 @@ class GcPresenceReceivedEvent(nec.NetworkIncomingEvent, HelperEvent): if jid: # we know real jid, save it in db st += ' (%s)' % jid - gajim.logger.write('gcstatus', self.fjid, st, self.show) + show = gajim.logger.convert_show_values_to_db_api_values(self.show) + if show is not None: + fjid = nbxmpp.JID(self.fjid) + gajim.logger.insert_into_logs(fjid.getStripped(), + time_time(), + KindConstant.GCSTATUS, + contact_name=fjid.getResource(), + message=st, + show=show) if self.avatar_sha == '': # contact has no avatar puny_nick = helpers.sanitize_filename(self.nick) @@ -1137,7 +1145,7 @@ class MessageReceivedEvent(nec.NetworkIncomingEvent, HelperEvent): base_network_events = ['raw-message-received']

def init(self): - self.additional_data = {} + self.additional_data = None

def generate(self): self.conn = self.base_event.conn

===================================== gajim/common/logger.py ===================================== --- a/gajim/common/logger.py +++ b/gajim/common/logger.py @@ -122,8 +122,6 @@ class Logger: # db will be created in src/common/checks_paths.py return self.attach_cache_database() - gajim.ged.register_event_handler('gc-message-received', - ged.POSTCORE, self._nec_gc_message_received)

@staticmethod def namedtuple_factory(cursor, row): @@ -328,7 +326,7 @@ class Logger: self.jids_already_in.append(jid) return jid_id

- def convert_human_values_to_db_api_values(self, kind, show): + def convert_kind_values_to_db_api_values(self, kind): """ Convert from string style to constant ints for db """ @@ -349,25 +347,30 @@ class Logger: elif kind == 'error': kind_col = KindConstant.ERROR

+ return kind_col + + def convert_show_values_to_db_api_values(self, show): + """ + Convert from string style to constant ints for db + """ + if show == 'online': - show_col = ShowConstant.ONLINE + return ShowConstant.ONLINE elif show == 'chat': - show_col = ShowConstant.CHAT + return ShowConstant.CHAT elif show == 'away': - show_col = ShowConstant.AWAY + return ShowConstant.AWAY elif show == 'xa': - show_col = ShowConstant.XA + return ShowConstant.XA elif show == 'dnd': - show_col = ShowConstant.DND + return ShowConstant.DND elif show == 'offline': - show_col = ShowConstant.OFFLINE + return ShowConstant.OFFLINE elif show is None: - show_col = None + return ShowConstant.ONLINE else: # invisible in GC when someone goes invisible # it's a RFC violation .... but we should not crash - show_col = 'UNKNOWN' - - return kind_col, show_col + return None

def convert_human_transport_type_to_db_api_values(self, type_): """ @@ -466,28 +469,6 @@ class Logger: if sub == SubscriptionConstant.BOTH: return 'both'

- def commit_to_db(self, values, write_unread=False): - sql = '''INSERT INTO logs (jid_id, contact_name, time, kind, show, - message, subject, additional_data) VALUES (?, ?, ?, ?, ?, ?, ?, ?)''' - try: - self.cur.execute(sql, values) - except sqlite.OperationalError as e: - raise exceptions.PysqliteOperationalError(str(e)) - except sqlite.DatabaseError: - raise exceptions.DatabaseMalformed(LOG_DB_PATH) - message_id = None - if write_unread: - try: - self.con.commit() - message_id = self.cur.lastrowid - except sqlite.OperationalError as e: - print(str(e), file=sys.stderr) - else: - self._timeout_commit() - if message_id: - self.insert_unread_events(message_id, values[0]) - return message_id - def insert_unread_events(self, message_id, jid_id): """ Add unread message with id: message_id @@ -552,89 +533,6 @@ class Logger: all_messages.append((results, shown)) return all_messages

- def write(self, kind, jid, message=None, show=None, tim=None, subject=None, - additional_data=None, mam_query=False): - """ - Write a row (status, gcstatus, message etc) to logs database - - kind can be status, gcstatus, gc_msg, (we only recv for those 3), - single_msg_recv, chat_msg_recv, chat_msg_sent, single_msg_sent we cannot - know if it is pm or normal chat message, we try to guess see - jid_is_from_pm() - - We analyze jid and store it as follows: - jids.jid text column will hold JID if TC-related, room_jid if GC-related, - ROOM_JID/nick if pm-related. - """ - - if additional_data is None: - additional_data = {} - if self.jids_already_in == []: # only happens if we just created the db - self.open_db() - - contact_name_col = None # holds nickname for kinds gcstatus, gc_msg - # message holds the message unless kind is status or gcstatus, - # then it holds status message - message_col = message - subject_col = subject - additional_data_col = json.dumps(additional_data) - if tim: - time_col = float(tim) - else: - time_col = float(time.time()) - - kind_col, show_col = self.convert_human_values_to_db_api_values(kind, - show) - - write_unread = False - try: - # now we may have need to do extra care for some values in columns - if kind == 'status': # we store (not None) time, jid, show, msg - # status for roster items - jid_id = self.get_jid_id(jid) - if show is None: # show is None (xmpp), but we say that 'online' - show_col = ShowConstant.ONLINE - - elif kind == 'gcstatus': - # status in ROOM (for pm status see status) - if show is None: # show is None (xmpp), but we say that 'online' - show_col = ShowConstant.ONLINE - jid, nick = jid.split('/', 1) - - # re-get jid_id for the new jid - jid_id = self.get_jid_id(jid, 'ROOM') - contact_name_col = nick - - elif kind == 'gc_msg': - if jid.find('/') != -1: # if it has a / - jid, nick = jid.split('/', 1) - else: - # it's server message f.e. error message - # when user tries to ban someone but he's not allowed to - nick = None - - # re-get jid_id for the new jid - jid_id = self.get_jid_id(jid, 'ROOM') - - contact_name_col = nick - else: - jid_id = self.get_jid_id(jid) - if kind == 'chat_msg_recv': - if not self.jid_is_from_pm(jid) and not mam_query: - # Save in unread table only if it's not a pm - write_unread = True - - if show_col == 'UNKNOWN': # unknown show, do not log - return - - values = (jid_id, contact_name_col, time_col, kind_col, show_col, - message_col, subject_col, additional_data_col) - return self.commit_to_db(values, write_unread) - - except (exceptions.DatabaseMalformed, - exceptions.PysqliteOperationalError) as error: - self.dispatch('DB_ERROR', error) - def get_last_conversation_lines(self, account, jid, pending): """ Get recent messages @@ -854,16 +752,21 @@ class Logger: return self.get_last_date_that_has_logs(account, jid) return row.time

- def set_room_last_message_time(self, jid, time): + def set_room_last_message_time(self, jid, timestamp): """ - Set last time (in seconds since EPOCH) for which we had logs for that - room in rooms_last_message_time table + Set the timestamp of the last message we received in a room. + + :param jid: The jid + + :param timestamp: The timestamp in epoch """ - jid_id = self.get_jid_id(jid, 'ROOM') - # jid_id is unique in this table, create or update : - sql = 'REPLACE INTO rooms_last_message_time VALUES (%d, %d)' % \ - (jid_id, time) - self.simple_commit(sql) + self.insert_jid(jid, type_=JIDConstant.ROOM_TYPE) + + sql = '''REPLACE INTO rooms_last_message_time + VALUES ((SELECT jid_id FROM jids WHERE jid = ?), ?)''' + + self.con.execute(sql, (jid, timestamp)) + self._timeout_commit()

def save_transport_type(self, jid, type_): """ @@ -1130,14 +1033,19 @@ class Logger:

def remove_roster(self, account_jid): """ - Remove all entry from account_jid roster + Remove the roster of an account + + :param account_jid: The jid of the account """ - account_jid_id = self.get_jid_id(account_jid)

- self.cur.execute('DELETE FROM roster_entry WHERE account_jid_id=?', - (account_jid_id,)) - self.cur.execute('DELETE FROM roster_group WHERE account_jid_id=?', - (account_jid_id,)) + jid_id = self.get_jid_id(account_jid) + + sql = ''' + DELETE FROM roster_entry WHERE account_jid_id = {jid_id}; + DELETE FROM roster_group WHERE account_jid_id = {jid_id}; + '''.format(jid_id=jid_id) + + self.con.executescript(sql) self._timeout_commit()

def search_for_duplicate(self, jid, timestamp, msg): @@ -1181,10 +1089,13 @@ class Logger:

:param type_: A JIDConstant """ - if kind == KindConstant.GC_MSG: + if jid in self.jids_already_in: + return + if kind in (KindConstant.GC_MSG, KindConstant.GCSTATUS): type_ = JIDConstant.ROOM_TYPE sql = 'INSERT OR IGNORE INTO jids (jid, type) VALUES (?, ?)' self.con.execute(sql, (jid, type_)) + self.jids_already_in.append(jid) self._timeout_commit()

def insert_into_logs(self, jid, time_, kind, unread=True, **kwargs): @@ -1221,18 +1132,3 @@ class Logger: self._timeout_commit()

return lastrowid - - def _nec_gc_message_received(self, obj): - tim_f = float(obj.timestamp) - tim_int = int(tim_f) - if gajim.config.should_log(obj.conn.name, obj.jid) and not \ - tim_int < obj.conn.last_history_time[obj.jid] and obj.msgtxt and \ - obj.nick: - # if not obj.nick, it means message comes from room itself - # usually it hold description and can be send at each connection - # so don't store it in logs - self.write('gc_msg', obj.fjid, obj.msgtxt, tim=obj.timestamp, additional_data=obj.additional_data) - # store in memory time of last message logged. - # this will also be saved in rooms_last_message_time table - # when we quit this muc - obj.conn.last_history_time[obj.jid] = tim_f

===================================== gajim/session.py ===================================== --- a/gajim/session.py +++ b/gajim/session.py @@ -21,6 +21,8 @@ ## along with Gajim. If not, see http://www.gnu.org/licenses/. ##

+import nbxmpp + from common import helpers

from common import events @@ -31,6 +33,7 @@ from common import contacts from common import ged from common.connection_handlers_events import ChatstateReceivedEvent, \ InformationEvent +from common.logger import KindConstant

import message_control

@@ -86,21 +89,13 @@ class ChatControlSession(stanza_session.EncryptedStanzaSession): if not obj.msgtxt and obj.chatstate is None: return

- log_type = 'chat_msg' + log_type = KindConstant.CHAT_MSG_RECV + if obj.forwarded and obj.sent: + log_type = KindConstant.CHAT_MSG_SENT else: - log_type = 'single_msg' - end = '_recv' - if obj.forwarded and obj.sent: - end = '_sent' - log_type += end - - if self.is_loggable() and obj.msgtxt: - if obj.xhtml and gajim.config.get('log_xhtml_messages'): - msg_to_log = obj.xhtml - else: - msg_to_log = obj.msgtxt - obj.msg_log_id = gajim.logger.write(log_type, obj.fjid, - msg_to_log, tim=obj.timestamp, subject=obj.subject, additional_data=obj.additional_data) + log_type = KindConstant.SINGLE_MSG_RECV + if obj.forwarded and obj.sent: + log_type = KindConstant.SINGLE_MSG_SENT

treat_as = gajim.config.get('treat_incoming_messages') if treat_as: @@ -111,6 +106,22 @@ class ChatControlSession(stanza_session.EncryptedStanzaSession): pm = True obj.mtype = 'pm'

+ if self.is_loggable() and obj.msgtxt: + if obj.xhtml and gajim.config.get('log_xhtml_messages'): + msg_to_log = obj.xhtml + else: + msg_to_log = obj.msgtxt + + jid = obj.fjid + if not pm: + jid = obj.jid + + obj.msg_log_id = gajim.logger.insert_into_logs( + jid, obj.timestamp, log_type, + message=msg_to_log, + subject=obj.subject, + additional_data=obj.additional_data) + # Handle chat states if contact and (not obj.forwarded or not obj.sent): if self.control and self.control.type_id == \

View it on GitLab: https://dev.gajim.org/gajim/gajim/compare/635ce52856d4a71f4b2f9928e30bbe89d2...

--- View it on GitLab: https://dev.gajim.org/gajim/gajim/compare/635ce52856d4a71f4b2f9928e30bbe89d2... You're receiving this email because of your account on dev.gajim.org.