Philipp Hörist pushed to branch master at gajim / gajim
Commits: 1eee7285 by lovetox at 2020-09-17T22:19:19+02:00 Refactor cache storage
- Add Storage base class - Move Cache code into own module - Simplify roster cache
- - - - -
23 changed files:
- gajim/application.py - gajim/common/app.py - gajim/common/contacts.py - gajim/common/helpers.py - gajim/common/logger.py - gajim/common/modules/caps.py - gajim/common/modules/discovery.py - gajim/common/modules/mam.py - gajim/common/modules/message.py - gajim/common/modules/muc.py - gajim/common/modules/roster.py - gajim/common/modules/user_avatar.py - gajim/common/modules/vcard_avatars.py - gajim/common/modules/vcard_temp.py - + gajim/common/storage/__init__.py - + gajim/common/storage/base.py - + gajim/common/storage/cache.py - gajim/groupchat_control.py - gajim/gtk/avatar.py - gajim/gtk/groupchat_config.py - gajim/gtk/tooltips.py - gajim/gui_interface.py - gajim/roster_window.py
Changes:
===================================== gajim/application.py ===================================== @@ -56,6 +56,7 @@ from gajim.common import logger from gajim.common.i18n import _ from gajim.common.contacts import LegacyContactsAPI from gajim.common.task_manager import TaskManager +from gajim.common.storage.cache import CacheStorage
class GajimApplication(Gtk.Application): @@ -182,6 +183,8 @@ class GajimApplication(Gtk.Application): app.print_version() app.detect_dependencies() configpaths.create_paths() + app.storage.cache = CacheStorage() + app.storage.cache.init() try: app.logger = logger.Logger() app.contacts = LegacyContactsAPI()
===================================== gajim/common/app.py ===================================== @@ -71,6 +71,12 @@ plugin_manager = None # Plugins Manager
logger = cast(LoggerT, None)
+class Storage: + def __init__(self): + self.cache = None + +storage = Storage() + css_config = None
transport_type = {} # type: Dict[str, str]
===================================== gajim/common/contacts.py ===================================== @@ -143,7 +143,7 @@ class CommonContact(XMPPEntity): # return caps for a contact that has no resources left. return False
- disco_info = app.logger.get_last_disco_info(self.get_full_jid()) + disco_info = app.storage.cache.get_last_disco_info(self.get_full_jid()) if disco_info is None: return False
@@ -151,7 +151,7 @@ class CommonContact(XMPPEntity):
@property def uses_phone(self): - disco_info = app.logger.get_last_disco_info(self.get_full_jid()) + disco_info = app.storage.cache.get_last_disco_info(self.get_full_jid()) if disco_info is None: return False
@@ -271,7 +271,7 @@ class Contact(CommonContact): if not self.is_groupchat: raise ValueError
- disco_info = app.logger.get_last_disco_info(self.jid) + disco_info = app.storage.cache.get_last_disco_info(self.jid)
context = 'public' if disco_info is not None and disco_info.muc_is_members_only:
===================================== gajim/common/helpers.py ===================================== @@ -898,7 +898,7 @@ def call_counter(func): return helper
def get_sync_threshold(jid, archive_info): - disco_info = app.logger.get_last_disco_info(jid) + disco_info = app.storage.cache.get_last_disco_info(jid) if archive_info is None or archive_info.sync_threshold is None: if disco_info is not None and disco_info.muc_is_members_only: threshold = app.settings.get('private_room_sync_threshold') @@ -1231,7 +1231,7 @@ def get_groupchat_name(con, jid): if name: return name
- disco_info = app.logger.get_last_disco_info(jid) + disco_info = app.storage.cache.get_last_disco_info(jid) if disco_info is not None: if disco_info.muc_name: return disco_info.muc_name
===================================== gajim/common/logger.py ===================================== @@ -28,7 +28,6 @@ This module allows to access the on-disk database of logs import os import sys import time -import math import datetime import calendar import json @@ -38,20 +37,16 @@ from collections import namedtuple
from gi.repository import GLib
-from nbxmpp.protocol import Iq -from nbxmpp.protocol import JID -from nbxmpp.structs import DiscoInfo -from nbxmpp.structs import CommonError -from nbxmpp.modules.discovery import parse_disco_info - from gajim.common import exceptions from gajim.common import app from gajim.common import configpaths from gajim.common.helpers import AdditionalDataDict from gajim.common.i18n import _ -from gajim.common.const import ( - JIDConstant, KindConstant, ShowConstant, TypeConstant, - SubscriptionConstant) +from gajim.common.const import ShowConstant +from gajim.common.const import KindConstant +from gajim.common.const import JIDConstant + +from gajim.common.storage.base import timeit
LOGS_SQL_STATEMENT = ''' @@ -97,113 +92,23 @@ LOGS_SQL_STATEMENT = ''' PRAGMA user_version=5; '''
-CACHE_SQL_STATEMENT = ''' - CREATE TABLE transports_cache ( - transport TEXT UNIQUE, - type INTEGER - ); - CREATE TABLE caps_cache ( - hash_method TEXT, - hash TEXT, - data TEXT, - last_seen INTEGER - ); - CREATE TABLE last_seen_disco_info( - jid TEXT PRIMARY KEY UNIQUE, - disco_info TEXT, - last_seen INTEGER - ); - CREATE TABLE roster_entry( - account_jid_id INTEGER, - jid_id INTEGER, - name TEXT, - subscription INTEGER, - ask BOOLEAN, - avatar_sha TEXT, - PRIMARY KEY (account_jid_id, jid_id) - ); - CREATE TABLE roster_group( - account_jid_id INTEGER, - jid_id INTEGER, - group_name TEXT, - PRIMARY KEY (account_jid_id, jid_id, group_name) - ); - CREATE TABLE muc_avatars( - jid TEXT PRIMARY KEY UNIQUE, - avatar_sha TEXT - ); - PRAGMA user_version=5; - '''
log = logging.getLogger('gajim.c.logger')
-def timeit(func): - def func_wrapper(self, *args, **kwargs): - start = time.time() - result = func(self, *args, **kwargs) - exec_time = (time.time() - start) * 1e3 - level = 30 if exec_time > 50 else 10 - log.log(level, 'Execution time for %s: %s ms', - func.__name__, math.ceil(exec_time)) - return result - return func_wrapper - -def _convert_disco_info(disco_info): - return parse_disco_info(Iq(node=disco_info)) - -def _adapt_disco_info(disco_info): - return str(disco_info.stanza) - -def _convert_common_error(common_error): - return CommonError.from_string(common_error) - -def _adapt_common_error(common_error): - return common_error.serialize() - -def _convert_marker(marker): - return 'received' if marker == 0 else 'displayed' - -def _jid_adapter(jid): - return str(jid) - -def _jid_converter(jid): - return JID.from_string(jid.decode()) - - -sqlite.register_converter('jid', _jid_converter) -sqlite.register_adapter(JID, _jid_adapter) - -sqlite.register_converter('disco_info', _convert_disco_info) -sqlite.register_adapter(DiscoInfo, _adapt_disco_info) - -sqlite.register_converter('common_error', _convert_common_error) -sqlite.register_adapter(CommonError, _adapt_common_error) - -sqlite.register_converter('marker', _convert_marker) - - class Logger: def __init__(self): + self._log = log self._jid_ids = {} self._jid_ids_reversed = {} self._con = None self._commit_timout_id = None self._log_db_path = configpaths.get('LOG_DB') - self._cache_db_path = configpaths.get('CACHE_DB') - - self._entity_caps_cache = {} - self._disco_info_cache = {} - self._muc_avatar_sha_cache = {}
self._create_databases() self._migrate_databases() self._connect_databases() self._get_jid_ids_from_db() - self._fill_disco_info_cache() - self._fill_muc_avatar_sha_cache() - self._clean_caps_table() - self._load_caps_data()
def _create_databases(self): if os.path.isdir(self._log_db_path): @@ -211,19 +116,9 @@ class Logger: self._log_db_path) sys.exit()
- if os.path.isdir(self._cache_db_path): - log.error(_('%s is a directory but should be a file'), - self._cache_db_path) - sys.exit() - if not os.path.exists(self._log_db_path): - if os.path.exists(self._cache_db_path): - os.remove(self._cache_db_path) self._create(LOGS_SQL_STATEMENT, self._log_db_path)
- if not os.path.exists(self._cache_db_path): - self._create(CACHE_SQL_STATEMENT, self._cache_db_path) - @staticmethod def _connect(*args, **kwargs): con = sqlite.connect(*args, **kwargs) @@ -257,10 +152,6 @@ class Logger: con = self._connect(self._log_db_path) self._migrate_logs(con) con.close() - - con = self._connect(self._cache_db_path) - self._migrate_cache(con) - con.close() except Exception: log.exception('Error') sys.exit() @@ -318,50 +209,6 @@ class Logger: ] self._execute_multiple(con, statements)
- @timeit - def _migrate_cache(self, con): - if self._get_user_version(con) == 0: - # All migrations from 0.16.9 until 1.0.0 - statements = [ - 'ALTER TABLE roster_entry ADD COLUMN "avatar_sha" TEXT', - 'PRAGMA user_version=1' - ] - self._execute_multiple(con, statements) - - if self._get_user_version(con) < 2: - statements = [ - 'DROP TABLE IF EXISTS caps_cache', - 'CREATE TABLE caps_cache (hash_method TEXT, hash TEXT, data TEXT, last_seen INTEGER)', - 'PRAGMA user_version=2' - ] - self._execute_multiple(con, statements) - - if self._get_user_version(con) < 3: - statements = [ - '''CREATE TABLE last_seen_disco_info( - jid TEXT PRIMARY KEY UNIQUE, - disco_info TEXT, - last_seen INTEGER)''', - 'PRAGMA user_version=3' - ] - self._execute_multiple(con, statements) - - if self._get_user_version(con) < 4: - statements = [ - '''CREATE TABLE muc_avatars( - jid TEXT PRIMARY KEY UNIQUE, - avatar_sha TEXT)''', - 'PRAGMA user_version=4' - ] - self._execute_multiple(con, statements) - - if self._get_user_version(con) < 5: - statements = [ - 'DELETE FROM caps_cache', - 'PRAGMA user_version=5' - ] - self._execute_multiple(con, statements) - @staticmethod def _execute_multiple(con, statements): """ @@ -416,13 +263,6 @@ class Logger: self._con.create_function("get_timeout", 0, self._get_timeout)
self._set_synchronous(False) - try: - self._con.execute("ATTACH DATABASE '%s' AS cache" % - self._cache_db_path.replace("'", "''")) - except Exception: - log.exception('Error') - self._con.close() - sys.exit()
@timeit def _set_synchronous(self, sync): @@ -594,99 +434,6 @@ class Logger: # it's a RFC violation .... but we should not crash return None
- def convert_human_transport_type_to_db_api_values(self, type_): - """ - Convert from string style to constant ints for db - """ - if type_ == 'aim': - return TypeConstant.AIM - if type_ == 'gadu-gadu': - return TypeConstant.GG - if type_ == 'http-ws': - return TypeConstant.HTTP_WS - if type_ == 'icq': - return TypeConstant.ICQ - if type_ == 'msn': - return TypeConstant.MSN - if type_ == 'qq': - return TypeConstant.QQ - if type_ == 'sms': - return TypeConstant.SMS - if type_ == 'smtp': - return TypeConstant.SMTP - if type_ in ('tlen', 'x-tlen'): - return TypeConstant.TLEN - if type_ == 'newmail': - return TypeConstant.NEWMAIL - if type_ == 'rss': - return TypeConstant.RSS - if type_ == 'weather': - return TypeConstant.WEATHER - if type_ == 'mrim': - return TypeConstant.MRIM - if type_ == 'jabber': - return TypeConstant.NO_TRANSPORT - return None - - def convert_api_values_to_human_transport_type(self, type_id): - """ - Convert from constant ints for db to string style - """ - if type_id == TypeConstant.AIM: - return 'aim' - if type_id == TypeConstant.GG: - return 'gadu-gadu' - if type_id == TypeConstant.HTTP_WS: - return 'http-ws' - if type_id == TypeConstant.ICQ: - return 'icq' - if type_id == TypeConstant.MSN: - return 'msn' - if type_id == TypeConstant.QQ: - return 'qq' - if type_id == TypeConstant.SMS: - return 'sms' - if type_id == TypeConstant.SMTP: - return 'smtp' - if type_id == TypeConstant.TLEN: - return 'tlen' - if type_id == TypeConstant.NEWMAIL: - return 'newmail' - if type_id == TypeConstant.RSS: - return 'rss' - if type_id == TypeConstant.WEATHER: - return 'weather' - if type_id == TypeConstant.MRIM: - return 'mrim' - if type_id == TypeConstant.NO_TRANSPORT: - return 'jabber' - - def convert_xmpp_sub(self, sub): - """ - Convert from string style to constant ints for db - """ - if sub == 'none': - return SubscriptionConstant.NONE - if sub == 'to': - return SubscriptionConstant.TO - if sub == 'from': - return SubscriptionConstant.FROM - if sub == 'both': - return SubscriptionConstant.BOTH - - def convert_db_sub(self, sub): - """ - Convert from constant ints for db to string style - """ - if sub == SubscriptionConstant.NONE: - return 'none' - if sub == SubscriptionConstant.TO: - return 'to' - if sub == SubscriptionConstant.FROM: - return 'from' - if sub == SubscriptionConstant.BOTH: - return 'both' - @timeit def insert_unread_events(self, message_id, jid_id): """ @@ -1043,246 +790,6 @@ class Logger: return self._con.execute( sql, tuple(jids) + (start, end)).fetchone()
- @timeit - def save_transport_type(self, jid, type_): - """ - Save the type of the transport in DB - """ - type_id = self.convert_human_transport_type_to_db_api_values(type_) - if not type_id: - # unknown type - return - result = self._con.execute( - 'SELECT type from transports_cache WHERE transport = "%s"' % jid).fetchone() - if result: - if result.type == type_id: - return - sql = 'UPDATE transports_cache SET type = %d WHERE transport = "%s"' %\ - (type_id, jid) - self.simple_commit(sql) - return - sql = 'INSERT INTO transports_cache (transport, type) VALUES (?, ?)' - self._con.execute(sql, (jid, type_id)) - self._timeout_commit() - - @timeit - def get_transports_type(self): - """ - Return all the type of the transports in DB - """ - results = self._con.execute('SELECT * from transports_cache').fetchall() - if not results: - return {} - answer = {} - for result in results: - answer[result.transport] = self.convert_api_values_to_human_transport_type( - result.type) - return answer - - @timeit - def _load_caps_data(self): - ''' - Load caps cache data - ''' - rows = self._con.execute( - 'SELECT hash_method, hash, data as "data [disco_info]" ' - 'FROM caps_cache') - - for row in rows: - self._entity_caps_cache[(row.hash_method, row.hash)] = row.data - - @timeit - def add_caps_entry(self, jid, hash_method, hash_, caps_data): - self._entity_caps_cache[(hash_method, hash_)] = caps_data - - self._disco_info_cache[jid] = caps_data - - self._con.execute(''' - INSERT INTO caps_cache (hash_method, hash, data, last_seen) - VALUES (?, ?, ?, ?) - ''', (hash_method, hash_, caps_data, int(time.time()))) - self._timeout_commit() - - def get_caps_entry(self, hash_method, hash_): - return self._entity_caps_cache.get((hash_method, hash_)) - - @timeit - def update_caps_time(self, method, hash_): - sql = '''UPDATE caps_cache SET last_seen = ? - WHERE hash_method = ? and hash = ?''' - self._con.execute(sql, (int(time.time()), method, hash_)) - self._timeout_commit() - - @timeit - def _clean_caps_table(self): - """ - Remove caps which was not seen for 3 months - """ - timestamp = int(time.time()) - 3 * 30 * 24 * 3600 - self._con.execute('DELETE FROM caps_cache WHERE last_seen < ?', - (timestamp,)) - self._timeout_commit() - - @timeit - def replace_roster(self, account_name, roster_version, roster): - """ - Replace current roster in DB by a new one - - accout_name is the name of the account to change. - roster_version is the version of the new roster. - roster is the new version. - """ - # First we must reset roster_version value to ensure that the server - # sends back all the roster at the next connection if the replacement - # didn't work properly. - app.settings.set_account_setting(account_name, 'roster_version', '') - - account_jid = app.get_jid_from_account(account_name) - # Execute get_jid_id() because this ensures on new accounts that the - # jid_id will be created - self.get_jid_id(account_jid, type_=JIDConstant.NORMAL_TYPE) - - # Delete old roster - self.remove_roster(account_jid) - - # Fill roster tables with the new roster - for jid in roster: - self.add_or_update_contact(account_jid, jid, roster[jid]['name'], - roster[jid]['subscription'], roster[jid]['ask'], - roster[jid]['groups'], commit=False) - self._timeout_commit() - - # At this point, we are sure the replacement works properly so we can - # set the new roster_version value. - app.settings.set_account_setting(account_name, - 'roster_version', - roster_version) - - @timeit - def del_contact(self, account_jid, jid): - """ - Remove jid from account_jid roster - """ - try: - account_jid_id = self.get_jid_id(account_jid) - jid_id = self.get_jid_id(jid) - except exceptions.PysqliteOperationalError as e: - raise exceptions.PysqliteOperationalError(str(e)) - self._con.execute( - 'DELETE FROM roster_group WHERE account_jid_id=? AND jid_id=?', - (account_jid_id, jid_id)) - self._con.execute( - 'DELETE FROM roster_entry WHERE account_jid_id=? AND jid_id=?', - (account_jid_id, jid_id)) - self._timeout_commit() - - @timeit - def add_or_update_contact(self, account_jid, jid, name, sub, ask, groups, - commit=True): - """ - Add or update a contact from account_jid roster - """ - if sub == 'remove': - self.del_contact(account_jid, jid) - return - - try: - account_jid_id = self.get_jid_id(account_jid) - jid_id = self.get_jid_id(jid, type_=JIDConstant.NORMAL_TYPE) - except exceptions.PysqliteOperationalError as error: - raise exceptions.PysqliteOperationalError(str(error)) - - # Update groups information - # First we delete all previous groups information - sql = 'DELETE FROM roster_group WHERE account_jid_id=? AND jid_id=?' - self._con.execute(sql, (account_jid_id, jid_id)) - # Then we add all new groups information - sql = '''INSERT INTO roster_group (account_jid_id, jid_id, group_name) - VALUES (?, ?, ?)''' - for group in groups: - self._con.execute(sql, (account_jid_id, jid_id, group)) - - if name is None: - name = '' - - sql = '''REPLACE INTO roster_entry - (account_jid_id, jid_id, name, subscription, ask) - VALUES(?, ?, ?, ?, ?)''' - self._con.execute(sql, (account_jid_id, - jid_id, - name, - self.convert_xmpp_sub(sub), - bool(ask))) - if commit: - self._timeout_commit() - - @timeit - def get_roster(self, account_jid): - """ - Return the accound_jid roster in NonBlockingRoster format - """ - data = {} - account_jid_id = self.get_jid_id(account_jid, type_=JIDConstant.NORMAL_TYPE) - - # First we fill data with roster_entry information - rows = self._con.execute(''' - SELECT j.jid, re.jid_id, re.name, re.subscription, re.ask, re.avatar_sha - FROM roster_entry re, jids j - WHERE re.account_jid_id=? AND j.jid_id=re.jid_id''', (account_jid_id,)) - for row in rows: - #jid, jid_id, name, subscription, ask - jid = row.jid - name = row.name - data[jid] = {} - data[jid]['avatar_sha'] = row.avatar_sha - if name: - data[jid]['name'] = name - else: - data[jid]['name'] = None - data[jid]['subscription'] = self.convert_db_sub(row.subscription) - data[jid]['groups'] = [] - data[jid]['resources'] = {} - if row.ask: - data[jid]['ask'] = 'subscribe' - else: - data[jid]['ask'] = None - data[jid]['id'] = row.jid_id - - # Then we add group for roster entries - for jid in data: - rows = self._con.execute(''' - SELECT group_name FROM roster_group - WHERE account_jid_id=? AND jid_id=?''', - (account_jid_id, data[jid]['id'])) - for row in rows: - group_name = row.group_name - data[jid]['groups'].append(group_name) - del data[jid]['id'] - - return data - - @timeit - def remove_roster(self, account_jid): - """ - Remove the roster of an account - - :param account_jid: The jid of the account - """ - try: - jid_id = self.get_jid_id(account_jid) - except ValueError: - # This happens if the JID never made it to the Database - # because the account was never connected - return - - sql = ''' - DELETE FROM roster_entry WHERE account_jid_id = {jid_id}; - DELETE FROM roster_group WHERE account_jid_id = {jid_id}; - '''.format(jid_id=jid_id) - - self._con.executescript(sql) - self._timeout_commit() - @timeit def deduplicate_muc_message(self, account, jid, resource, timestamp, message_id): @@ -1519,71 +1026,6 @@ class Logger: self._con.execute(sql, (state, account_id, jid_id, message_id)) self._timeout_commit()
- @timeit - def _fill_muc_avatar_sha_cache(self): - sql = '''SELECT jid, avatar_sha FROM muc_avatars''' - rows = self._con.execute(sql).fetchall() - for row in rows: - self._muc_avatar_sha_cache[row.jid] = row.avatar_sha - log.info('%d Avatar SHA entries loaded', len(rows)) - - @timeit - def set_avatar_sha(self, account_jid, jid, sha=None): - """ - Set the avatar sha of a jid on an account - - :param account_jid: The jid of the account - - :param jid: The jid that belongs to the avatar - - :param sha: The sha of the avatar - - """ - - account_jid_id = self.get_jid_id(account_jid) - jid_id = self.get_jid_id(jid, type_=JIDConstant.NORMAL_TYPE) - - sql = ''' - UPDATE roster_entry SET avatar_sha = ? - WHERE account_jid_id = ? AND jid_id = ? - ''' - self._con.execute(sql, (sha, account_jid_id, jid_id)) - self._timeout_commit() - - @timeit - def set_muc_avatar_sha(self, jid, sha=None): - """ - Set the avatar sha of a MUC - - :param jid: The MUC jid that belongs to the avatar - - :param sha: The sha of the avatar - - """ - - sql = '''INSERT INTO muc_avatars (jid, avatar_sha) - VALUES (?, ?)''' - - try: - self._con.execute(sql, (jid, sha)) - except sqlite.IntegrityError: - sql = 'UPDATE muc_avatars SET avatar_sha = ? WHERE jid = ?' - self._con.execute(sql, (sha, jid)) - - self._muc_avatar_sha_cache[jid] = sha - - self._timeout_commit() - - def get_muc_avatar_sha(self, jid): - """ - Get the avatar sha of a MUC - - :param jid: The MUC jid that belongs to the avatar - - """ - - return self._muc_avatar_sha_cache.get(jid) - @timeit def get_archive_infos(self, jid): """ @@ -1657,66 +1099,3 @@ class Logger: self._con.execute(sql, (jid_id,)) log.info('Reset message archive info: %s', jid) self._timeout_commit() - - @timeit - def _fill_disco_info_cache(self): - sql = '''SELECT disco_info as "disco_info [disco_info]", - jid, last_seen FROM - last_seen_disco_info''' - rows = self._con.execute(sql).fetchall() - for row in rows: - disco_info = row.disco_info._replace(timestamp=row.last_seen) - self._disco_info_cache[row.jid] = disco_info - log.info('%d DiscoInfo entries loaded', len(rows)) - - def get_last_disco_info(self, jid, max_age=0): - """ - Get last disco info from jid - - :param jid: The jid - - :param max_age: max age in seconds of the DiscoInfo record - - """ - - disco_info = self._disco_info_cache.get(jid) - if disco_info is not None: - max_timestamp = time.time() - max_age if max_age else 0 - if max_timestamp > disco_info.timestamp: - return None - return disco_info - - @timeit - def set_last_disco_info(self, jid, disco_info, cache_only=False): - """ - Get last disco info from jid - - :param jid: The jid - - :param disco_info: A DiscoInfo object - - """ - - log.info('Save disco info from %s', jid) - - if cache_only: - self._disco_info_cache[jid] = disco_info - return - - disco_exists = self.get_last_disco_info(jid) is not None - if disco_exists: - sql = '''UPDATE last_seen_disco_info SET - disco_info = ?, last_seen = ? - WHERE jid = ?''' - - self._con.execute(sql, (disco_info, disco_info.timestamp, str(jid))) - - else: - sql = '''INSERT INTO last_seen_disco_info - (jid, disco_info, last_seen) - VALUES (?, ?, ?)''' - - self._con.execute(sql, (str(jid), disco_info, disco_info.timestamp)) - - self._disco_info_cache[jid] = disco_info - self._timeout_commit()
===================================== gajim/common/modules/caps.py ===================================== @@ -101,14 +101,14 @@ class Caps(BaseModule):
self._log.info('Received %s', task.entity)
- disco_info = app.logger.get_caps_entry(task.entity.method, - task.entity.hash) + disco_info = app.storage.cache.get_caps_entry(task.entity.method, + task.entity.hash) if disco_info is None: self._queue_task(task) return
jid = str(properties.jid) - app.logger.set_last_disco_info(jid, disco_info, cache_only=True) + app.storage.cache.set_last_disco_info(jid, disco_info, cache_only=True) app.nec.push_incoming_event( NetworkEvent('caps-update', account=self._account, @@ -143,7 +143,7 @@ class Caps(BaseModule): disco_info.jid, error) return
- app.logger.add_caps_entry( + app.storage.cache.add_caps_entry( str(disco_info.jid), task.entity.method, disco_info.get_caps_hash(),
===================================== gajim/common/modules/discovery.py ===================================== @@ -153,7 +153,6 @@ class Discovery(BaseModule): jid = str(info.jid) if jid not in app.transport_type: app.transport_type[jid] = identity.type - app.logger.save_transport_type(jid, identity.type)
if identity.type in self._con.available_transports: self._con.available_transports[identity.type].append(jid) @@ -200,7 +199,7 @@ class Discovery(BaseModule): def _muc_info_received(self, result, callback=None): self._log.info('MUC info received: %s', result.jid) if not is_error_result(result): - app.logger.set_last_disco_info(result.jid, result) + app.storage.cache.set_last_disco_info(result.jid, result) self._con.get_module('VCardAvatars').muc_disco_info_update(result) app.nec.push_incoming_event(NetworkEvent( 'muc-disco-update',
===================================== gajim/common/modules/mam.py ===================================== @@ -122,7 +122,7 @@ class MAM(BaseModule): archive_jid = properties.jid.bare timestamp = properties.timestamp
- disco_info = app.logger.get_last_disco_info(archive_jid) + disco_info = app.storage.cache.get_last_disco_info(archive_jid) if disco_info is None: # This is the case on MUC creation # After MUC configuration we receive a configuration change
===================================== gajim/common/modules/message.py ===================================== @@ -253,7 +253,7 @@ class Message(BaseModule): message_id=event.properties.id)
def _check_for_mam_compliance(self, room_jid, stanza_id): - disco_info = app.logger.get_last_disco_info(room_jid) + disco_info = app.storage.cache.get_last_disco_info(room_jid) if stanza_id is None and disco_info.mam_namespace == Namespace.MAM_2: self._log.warning('%s announces mam:2 without stanza-id', room_jid)
@@ -266,7 +266,7 @@ class Message(BaseModule): return None, None
if properties.type.is_groupchat: - disco_info = app.logger.get_last_disco_info( + disco_info = app.storage.cache.get_last_disco_info( properties.jid.bare)
if disco_info.mam_namespace != Namespace.MAM_2:
===================================== gajim/common/modules/muc.py ===================================== @@ -141,7 +141,8 @@ class MUC(BaseModule):
self._manager.add(muc_data)
- disco_info = app.logger.get_last_disco_info(muc_data.jid, max_age=60) + disco_info = app.storage.cache.get_last_disco_info(muc_data.jid, + max_age=60) if disco_info is None: self._con.get_module('Discovery').disco_muc( muc_data.jid,
===================================== gajim/common/modules/roster.py ===================================== @@ -45,8 +45,7 @@ class Roster(BaseModule):
def load_roster(self): self._log.info('Load from database') - account_jid = self._con.get_own_jid().bare - data = app.logger.get_roster(account_jid) + data = app.storage.cache.load_roster(self._account) if data: self.set_raw(data) for jid, item in self._data.items(): @@ -71,6 +70,9 @@ class Roster(BaseModule): roster=self._data.copy(), received_from_server=False))
+ def _store_roster(self): + app.storage.cache.store_roster(self._account, self._data) + def request_roster(self): version = None features = self._con.connection.features @@ -99,7 +101,10 @@ class Roster(BaseModule): version = self._parse_roster(stanza)
self._log.info('New version: %s', version) - app.logger.replace_roster(self._account, version, self._data) + self._store_roster() + app.settings.set_account_setting(self._account, + 'roster_version', + version)
received_from_server = True
@@ -135,10 +140,8 @@ class Roster(BaseModule): ask=attrs['ask'], groups=attrs['groups'], avatar_sha=None)) - account_jid = self._con.get_own_jid().bare - app.logger.add_or_update_contact( - account_jid, item.jid, attrs['name'], - attrs['subscription'], attrs['ask'], attrs['groups']) + + self._store_roster()
self._log.info('New version: %s', version) app.settings.set_account_setting(self._account, @@ -238,13 +241,8 @@ class Roster(BaseModule): 'subscription': 'none', 'groups': ['Not in contact list']} - account_jid = self._con.get_own_jid().bare - app.logger.add_or_update_contact( - account_jid, jid, - self._data[jid]['name'], - self._data[jid]['subscription'], - self._data[jid]['ask'], - self._data[jid]['groups']) + + self._store_roster()
def _get_item_data(self, jid, dataname): """ @@ -369,6 +367,13 @@ class Roster(BaseModule): 'avatar_sha': None }
+ def set_avatar_sha(self, jid, sha): + if jid not in self._data: + return + + self._data[jid]['avatar_sha'] = sha + self._store_roster() +
def get_instance(*args, **kwargs): return Roster(*args, **kwargs), 'Roster'
===================================== gajim/common/modules/user_avatar.py ===================================== @@ -40,13 +40,12 @@ class UserAvatar(BaseModule):
data = properties.pubsub_event.data jid = str(properties.jid) - own_jid = self._con.get_own_jid().bare
if data is None: # Remove avatar self._log.info('Remove: %s', jid) app.contacts.set_avatar(self._account, jid, None) - app.logger.set_avatar_sha(own_jid, jid, None) + self._con.get_module('Roster').set_avatar_sha(jid, None) app.interface.update_avatar(self._account, jid) else: if properties.is_self_message: @@ -76,8 +75,8 @@ class UserAvatar(BaseModule): 'avatar_sha', result.sha) else: - own_jid = self._con.get_own_jid().bare - app.logger.set_avatar_sha(own_jid, str(result.jid), result.sha) + self._con.get_module('Roster').set_avatar_sha(str(result.jid), + result.sha)
app.contacts.set_avatar(self._account, str(result.jid), result.sha) app.interface.update_avatar(self._account, str(result.jid))
===================================== gajim/common/modules/vcard_avatars.py ===================================== @@ -132,7 +132,6 @@ class VCardAvatars(BaseModule): room)
def _process_update(self, jid, state, avatar_sha, room): - acc_jid = self._con.get_own_jid().bare if state == AvatarState.EMPTY: # Empty <photo/> tag, means no avatar is advertised self._log.info('%s has no avatar published', jid) @@ -142,9 +141,9 @@ class VCardAvatars(BaseModule): app.contacts.set_avatar(self._account, jid, None)
if room: - app.logger.set_muc_avatar_sha(jid, None) + app.storage.cache.set_muc_avatar_sha(jid, None) else: - app.logger.set_avatar_sha(acc_jid, jid, None) + self._con.get_module('Roster').set_avatar_sha(jid, None) app.interface.update_avatar(self._account, jid, room_avatar=room) else: self._log.info('Update: %s %s', jid, avatar_sha) @@ -158,9 +157,10 @@ class VCardAvatars(BaseModule): # Check if the avatar is already in storage self._log.info('Found avatar in storage') if room: - app.logger.set_muc_avatar_sha(jid, avatar_sha) + app.storage.cache.set_muc_avatar_sha(jid, avatar_sha) else: - app.logger.set_avatar_sha(acc_jid, jid, avatar_sha) + self._con.get_module('Roster').set_avatar_sha(jid, + avatar_sha) app.contacts.set_avatar(self._account, jid, avatar_sha) app.interface.update_avatar( self._account, jid, room_avatar=room)
===================================== gajim/common/modules/vcard_temp.py ===================================== @@ -266,7 +266,7 @@ class VCardTemp(BaseModule): app.interface.save_avatar(photo_decoded)
self._log.info('Received: %s %s', jid, avatar_sha) - app.logger.set_muc_avatar_sha(jid, avatar_sha) + app.storage.cache.set_muc_avatar_sha(jid, avatar_sha) app.contacts.set_avatar(self._account, jid, avatar_sha) app.interface.update_avatar(self._account, jid, room_avatar=True)
@@ -292,8 +292,7 @@ class VCardTemp(BaseModule): app.interface.update_avatar(contact=contact) else: self._log.info('Received: %s %s', jid, avatar_sha) - own_jid = self._con.get_own_jid().bare - app.logger.set_avatar_sha(own_jid, jid, avatar_sha) + self._con.get_module('Roster').set_avatar_sha(jid, avatar_sha) app.contacts.set_avatar(self._account, jid, avatar_sha) app.interface.update_avatar(self._account, jid)
===================================== gajim/common/storage/__init__.py =====================================
===================================== gajim/common/storage/base.py ===================================== @@ -0,0 +1,179 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see http://www.gnu.org/licenses/. + +import sys +import math +import time +import sqlite3 + +from gi.repository import GLib + +from nbxmpp.protocol import Iq +from nbxmpp.protocol import JID +from nbxmpp.structs import DiscoInfo +from nbxmpp.structs import CommonError +from nbxmpp.modules.discovery import parse_disco_info + + +def timeit(func): + def func_wrapper(self, *args, **kwargs): + start = time.time() + result = func(self, *args, **kwargs) + exec_time = (time.time() - start) * 1e3 + level = 30 if exec_time > 50 else 10 + self._log.log(level, + 'Execution time for %s: %s ms', + func.__name__, + math.ceil(exec_time)) + return result + return func_wrapper + + +def _convert_common_error(common_error): + return CommonError.from_string(common_error) + +def _adapt_common_error(common_error): + return common_error.serialize() + +sqlite3.register_converter('common_error', _convert_common_error) +sqlite3.register_adapter(CommonError, _adapt_common_error) + +def _convert_marker(marker): + return 'received' if marker == 0 else 'displayed' + +sqlite3.register_converter('marker', _convert_marker) + +def _jid_adapter(jid): + return str(jid) + +def _jid_converter(jid): + return JID.from_string(jid.decode()) + +sqlite3.register_converter('jid', _jid_converter) +sqlite3.register_adapter(JID, _jid_adapter) + +def _convert_disco_info(disco_info): + return parse_disco_info(Iq(node=disco_info)) + +def _adapt_disco_info(disco_info): + return str(disco_info.stanza) + +sqlite3.register_converter('disco_info', _convert_disco_info) +sqlite3.register_adapter(DiscoInfo, _adapt_disco_info) + + +class SqliteStorage: + ''' + Base Storage Class + ''' + + def __init__(self, + log, + path, + create_statement, + commit_delay=500): + + self._log = log + self._path = path + self._create_statement = create_statement + self._commit_delay = commit_delay + self._con = None + self._commit_source_id = None + + def init(self, **kwargs): + if self._path.exists(): + if not self._path.is_file(): + sys.exit('%s must be a file', self._path) + self._con = self._connect(**kwargs) + + else: + self._con = self._create_storage(**kwargs) + + self._migrate_storage() + + def _set_journal_mode(self, mode): + self._con.execute(f'PRAGMA journal_mode={mode}') + + def _set_synchronous(self, mode): + self._con.execute(f'PRAGMA synchronous={mode}') + + @property + def user_version(self) -> int: + return self._con.execute('PRAGMA user_version').fetchone()[0] + + def _connect(self, **kwargs): + return sqlite3.connect(self._path, **kwargs) + + def _create_storage(self, **kwargs): + self._log.info('Creating %s', self._path) + con = self._connect(**kwargs) + self._path.chmod(0o600) + + try: + con.executescript(self._create_statement) + except Exception: + self._log.exception('Error') + con.close() + self._path.unlink() + sys.exit('Failed creating storage') + + con.commit() + return con + + def _reinit_storage(self): + if self._con is not None: + self._con.close() + self._path.unlink() + self.init() + + def _migrate_storage(self): + try: + self._migrate() + except Exception: + self._con.close() + self._log.exception('Error') + sys.exit() + + def _migrate(self): + raise NotImplementedError + + def _execute_multiple(self, statements): + """ + Execute multiple statements with the option to fail on duplicates + but still continue + """ + for sql in statements: + try: + self._con.execute(sql) + self._con.commit() + except sqlite3.OperationalError as error: + if str(error).startswith('duplicate column name:'): + self._log.info(error) + else: + self._con.close() + self._log.exception('Error') + sys.exit() + + @timeit + def _commit(self): + self._commit_source_id = None + self._con.commit() + return False + + def _delayed_commit(self): + if self._commit_source_id is not None: + return + + self._commit_source_id = GLib.timeout_add(self._commit_delay, + self._commit)
===================================== gajim/common/storage/cache.py ===================================== @@ -0,0 +1,265 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see http://www.gnu.org/licenses/. + +import json +import time +import sqlite3 +import logging +from pathlib import Path +from collections import namedtuple + +from gajim.common import configpaths +from gajim.common.storage.base import SqliteStorage +from gajim.common.storage.base import timeit + + +CURRENT_USER_VERSION = 6 + +CACHE_SQL_STATEMENT = ''' + CREATE TABLE caps_cache ( + hash_method TEXT, + hash TEXT, + data TEXT, + last_seen INTEGER + ); + CREATE TABLE last_seen_disco_info( + jid TEXT PRIMARY KEY UNIQUE, + disco_info TEXT, + last_seen INTEGER + ); + CREATE TABLE roster( + account TEXT PRIMARY KEY UNIQUE, + roster TEXT + ); + CREATE TABLE muc_avatars( + jid TEXT PRIMARY KEY UNIQUE, + avatar_sha TEXT + ); + PRAGMA user_version=%s; + ''' % CURRENT_USER_VERSION + +log = logging.getLogger('gajim.c.storage.cache') + + +class CacheStorage(SqliteStorage): + def __init__(self): + SqliteStorage.__init__(self, + log, + Path(configpaths.get('CACHE_DB')), + CACHE_SQL_STATEMENT) + + self._entity_caps_cache = {} + self._disco_info_cache = {} + self._muc_avatar_sha_cache = {} + + def init(self, **kwargs): + SqliteStorage.init(self, + detect_types=sqlite3.PARSE_COLNAMES) + self._set_journal_mode('WAL') + self._con.row_factory = self._namedtuple_factory + + self._fill_disco_info_cache() + self._fill_muc_avatar_sha_cache() + self._clean_caps_table() + self._load_caps_data() + + @staticmethod + def _namedtuple_factory(cursor, row): + fields = [col[0] for col in cursor.description] + Row = namedtuple("Row", fields) + return Row(*row) + + def _migrate(self): + user_version = self.user_version + if user_version > CURRENT_USER_VERSION: + # Gajim was downgraded, reinit the storage + self._reinit_storage() + return + + if user_version < 6: + self._reinit_storage() + + @timeit + def _load_caps_data(self): + rows = self._con.execute( + 'SELECT hash_method, hash, data as "data [disco_info]" ' + 'FROM caps_cache') + + for row in rows: + self._entity_caps_cache[(row.hash_method, row.hash)] = row.data + + @timeit + def add_caps_entry(self, jid, hash_method, hash_, caps_data): + self._entity_caps_cache[(hash_method, hash_)] = caps_data + + self._disco_info_cache[jid] = caps_data + + self._con.execute(''' + INSERT INTO caps_cache (hash_method, hash, data, last_seen) + VALUES (?, ?, ?, ?) + ''', (hash_method, hash_, caps_data, int(time.time()))) + self._delayed_commit() + + def get_caps_entry(self, hash_method, hash_): + return self._entity_caps_cache.get((hash_method, hash_)) + + @timeit + def update_caps_time(self, method, hash_): + sql = '''UPDATE caps_cache SET last_seen = ? + WHERE hash_method = ? and hash = ?''' + self._con.execute(sql, (int(time.time()), method, hash_)) + self._delayed_commit() + + @timeit + def _clean_caps_table(self): + """ + Remove caps which was not seen for 3 months + """ + timestamp = int(time.time()) - 3 * 30 * 24 * 3600 + self._con.execute('DELETE FROM caps_cache WHERE last_seen < ?', + (timestamp,)) + self._delayed_commit() + + @timeit + def _fill_disco_info_cache(self): + sql = '''SELECT disco_info as "disco_info [disco_info]", + jid, last_seen FROM + last_seen_disco_info''' + rows = self._con.execute(sql).fetchall() + for row in rows: + disco_info = row.disco_info._replace(timestamp=row.last_seen) + self._disco_info_cache[row.jid] = disco_info + log.info('%d DiscoInfo entries loaded', len(rows)) + + def get_last_disco_info(self, jid, max_age=0): + """ + Get last disco info from jid + + :param jid: The jid + + :param max_age: max age in seconds of the DiscoInfo record + + """ + + disco_info = self._disco_info_cache.get(jid) + if disco_info is not None: + max_timestamp = time.time() - max_age if max_age else 0 + if max_timestamp > disco_info.timestamp: + return None + return disco_info + + @timeit + def set_last_disco_info(self, jid, disco_info, cache_only=False): + """ + Get last disco info from jid + + :param jid: The jid + + :param disco_info: A DiscoInfo object + + """ + + log.info('Save disco info from %s', jid) + + if cache_only: + self._disco_info_cache[jid] = disco_info + return + + disco_exists = self.get_last_disco_info(jid) is not None + if disco_exists: + sql = '''UPDATE last_seen_disco_info SET + disco_info = ?, last_seen = ? + WHERE jid = ?''' + + self._con.execute(sql, (disco_info, disco_info.timestamp, str(jid))) + + else: + sql = '''INSERT INTO last_seen_disco_info + (jid, disco_info, last_seen) + VALUES (?, ?, ?)''' + + self._con.execute(sql, (str(jid), disco_info, disco_info.timestamp)) + + self._disco_info_cache[jid] = disco_info + self._delayed_commit() + + @timeit + def store_roster(self, account, roster): + serialized = json.dumps(roster) + + insert_sql = 'INSERT INTO roster(account, roster) VALUES(?, ?)' + update_sql = 'UPDATE roster SET roster = ? WHERE account = ?' + + try: + self._con.execute(insert_sql, (account, serialized)) + except sqlite3.IntegrityError: + self._con.execute(update_sql, (serialized, account)) + + self._delayed_commit() + + @timeit + def load_roster(self, account): + select_sql = 'SELECT roster FROM roster WHERE account = ?' + result = self._con.execute(select_sql, (account,)).fetchone() + if result is None: + return None + return json.loads(result.roster) + + @timeit + def remove_roster(self, account): + delete_sql = 'DELETE FROM roster WHERE account = ?' + self._con.execute(delete_sql, (account,)) + self._commit() + + @timeit + def _fill_muc_avatar_sha_cache(self): + sql = '''SELECT jid, avatar_sha FROM muc_avatars''' + rows = self._con.execute(sql).fetchall() + for row in rows: + self._muc_avatar_sha_cache[row.jid] = row.avatar_sha + log.info('%d Avatar SHA entries loaded', len(rows)) + + @timeit + def set_muc_avatar_sha(self, jid, sha=None): + """ + Set the avatar sha of a MUC + + :param jid: The MUC jid that belongs to the avatar + + :param sha: The sha of the avatar + + """ + + sql = '''INSERT INTO muc_avatars (jid, avatar_sha) + VALUES (?, ?)''' + + try: + self._con.execute(sql, (jid, sha)) + except sqlite3.IntegrityError: + sql = 'UPDATE muc_avatars SET avatar_sha = ? WHERE jid = ?' + self._con.execute(sql, (sha, jid)) + + self._muc_avatar_sha_cache[jid] = sha + + self._delayed_commit() + + def get_muc_avatar_sha(self, jid): + """ + Get the avatar sha of a MUC + + :param jid: The MUC jid that belongs to the avatar + + """ + + return self._muc_avatar_sha_cache.get(jid)
===================================== gajim/groupchat_control.py ===================================== @@ -266,7 +266,7 @@ class GroupchatControl(ChatControlBase):
@property def disco_info(self): - return app.logger.get_last_disco_info(self.contact.jid) + return app.storage.cache.get_last_disco_info(self.contact.jid)
def add_actions(self): super().add_actions()
===================================== gajim/gtk/avatar.py ===================================== @@ -239,7 +239,7 @@ class AvatarStorage(metaclass=Singleton): if surface is not None: return surface
- avatar_sha = app.logger.get_muc_avatar_sha(jid) + avatar_sha = app.storage.cache.get_muc_avatar_sha(jid) if avatar_sha is not None: surface = self.surface_from_filename(avatar_sha, size, scale) if surface is None:
===================================== gajim/gtk/groupchat_config.py ===================================== @@ -52,7 +52,7 @@ class GroupchatConfig(Gtk.ApplicationWindow): self._ui.add_button.set_sensitive(True) self._ui.add_button.set_tooltip_text('')
- disco_info = app.logger.get_last_disco_info(self.jid) + disco_info = app.storage.cache.get_last_disco_info(self.jid) visible = disco_info.supports(Namespace.REGISTER) self._ui.reserved_name_column.set_visible(visible) self._ui.info_button.set_sensitive(False)
===================================== gajim/gtk/tooltips.py ===================================== @@ -357,7 +357,7 @@ class RosterTooltip(StatusTable):
else: # only one resource if contact.is_groupchat: - disco_info = app.logger.get_last_disco_info(contact.jid) + disco_info = app.storage.cache.get_last_disco_info(contact.jid) if disco_info is not None: description = disco_info.muc_description if description:
===================================== gajim/gui_interface.py ===================================== @@ -1321,7 +1321,7 @@ class Interface:
def create_groupchat_control(self, account, room_jid, muc_data, minimize=False): - avatar_sha = app.logger.get_muc_avatar_sha(room_jid) + avatar_sha = app.storage.cache.get_muc_avatar_sha(room_jid) contact = app.contacts.create_contact(jid=room_jid, account=account, groups=[_('Group chats')], @@ -1653,7 +1653,7 @@ class Interface: if app.settings.get_account_setting(account, 'active'): self.disable_account(account)
- app.logger.remove_roster(app.get_jid_from_account(account)) + app.storage.cache.remove_roster(account) # Delete password must be before del_per() because it calls set_per() # which would recreate the account with defaults values if not found passwords.delete_password(account) @@ -2103,9 +2103,6 @@ class Interface:
self.make_regexps()
- # get transports type from DB - app.transport_type = app.logger.get_transports_type() - self.last_ftwindow_update = 0
self._network_monitor = Gio.NetworkMonitor.get_default()
===================================== gajim/roster_window.py ===================================== @@ -1155,7 +1155,7 @@ class RosterWindow: status_span = '\n<span size="small" style="italic" ' \ 'alpha="70%">{}</span>' if contact.is_groupchat: - disco_info = app.logger.get_last_disco_info(contact.jid) + disco_info = app.storage.cache.get_last_disco_info(contact.jid) if disco_info is not None: description = disco_info.muc_description if description:
View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/1eee7285475b1166fc6315ef3d114775c...
-
Philipp Hörist