Commits

commits@gajim.org

17632 discussions

[Git][gajim/gajim][master] Settings: Move treshold setting from archive storage
by Philipp Hörist 18 Sep '20

18 Sep '20

Philipp Hörist pushed to branch master at gajim / gajim Commits: a72de039 by lovetox at 2020-09-18T23:45:29+02:00 Settings: Move treshold setting from archive storage - - - - - 9 changed files: - gajim/common/config.py - gajim/common/const.py - gajim/common/helpers.py - gajim/common/modules/mam.py - gajim/common/setting_values.py - gajim/common/storage/archive.py - gajim/groupchat_control.py - gajim/gtk/preferences.py - gajim/gui_menu_builder.py Changes: ===================================== gajim/common/config.py ===================================== @@ -227,7 +227,6 @@ class Config: 'use_keyring': [opt_bool, True, _('If enabled, Gajim will use the System\'s Keyring to store account passwords.')], 'remote_commands': [opt_bool, False, _('If enabled, Gajim will execute XEP-0146 Commands.')], 'dark_theme': [opt_int, 2, _('2: System, 1: Enabled, 0: Disabled')], - 'threshold_options': [opt_str, '1, 2, 4, 10, 0', _('Options in days which can be chosen in the sync threshold menu'), True], 'public_room_sync_threshold': [opt_int, 1, _('Maximum history in days we request from a public group chat archive. 0: As much as possible.')], 'private_room_sync_threshold': [opt_int, 0, _('Maximum history in days we request from a private group chat archive. 0: As much as possible.')], 'show_subject_on_join': [opt_bool, True, _('If enabled, Gajim shows the group chat subject in the chat window when joining.')], ===================================== gajim/common/const.py ===================================== @@ -161,6 +161,7 @@ class Chatstate(IntEnum): class SyncThreshold(IntEnum): + NO_SYNC = -1 NO_THRESHOLD = 0 def __str__(self): @@ -1085,3 +1086,13 @@ URI_SCHEMES = { 'webcal://', 'aesgcm://', } + + +THRESHOLD_OPTIONS = { + -1: _('No Sync'), + 1: _('1 Day'), + 2: _('2 Days'), + 7: _('1 Week'), + 30: _('1 Month'), + 0: _('No Threshold'), +} ===================================== gajim/common/helpers.py ===================================== @@ -897,17 +897,6 @@ def call_counter(func): return func(self) return helper -def get_sync_threshold(jid, archive_info): - disco_info = app.storage.cache.get_last_disco_info(jid) - if archive_info is None or archive_info.sync_threshold is None: - if disco_info is not None and disco_info.muc_is_members_only: - threshold = app.settings.get('private_room_sync_threshold') - else: - threshold = app.settings.get('public_room_sync_threshold') - app.storage.archive.set_archive_infos(jid, sync_threshold=threshold) - return threshold - return archive_info.sync_threshold - def load_json(path, key=None, default=None): try: with path.open('r') as file: ===================================== gajim/common/modules/mam.py ===================================== @@ -30,7 +30,6 @@ from gajim.common.nec import NetworkIncomingEvent from gajim.common.const import ArchiveState from gajim.common.const import KindConstant from gajim.common.const import SyncThreshold -from gajim.common.helpers import get_sync_threshold from gajim.common.helpers import AdditionalDataDict from gajim.common.modules.misc import parse_oob from gajim.common.modules.misc import parse_correction @@ -302,9 +301,21 @@ class MAM(BaseModule): def request_archive_on_muc_join(self, jid): archive = app.storage.archive.get_archive_infos(jid) - threshold = get_sync_threshold(jid, archive) + disco_info = app.storage.cache.get_last_disco_info(jid) + + context = 'public' + if disco_info is not None and disco_info.muc_is_members_only: + context = 'private' + + threshold = app.settings.get_group_chat_setting(self._account, + jid, + 'sync_threshold', + context=context) self._log.info('Threshold for %s: %s', jid, threshold) + if threshold == SyncThreshold.NO_SYNC: + return + mam_id = None start_date = None if archive is None or archive.last_mam_id is None: ===================================== gajim/common/setting_values.py ===================================== @@ -167,9 +167,8 @@ APP_SETTINGS = { 'use_keyring': True, 'remote_commands': False, 'dark_theme': 2, - 'threshold_options': '1, 2, 4, 10, 0', - 'public_room_sync_threshold': 1, - 'private_room_sync_threshold': 0, + 'gc_sync_threshold_public_default': 1, + 'gc_sync_threshold_private_default': 0, 'show_subject_on_join': True, 'show_chatstate_in_roster': True, 'show_chatstate_in_tabs': True, @@ -275,6 +274,7 @@ ACCOUNT_SETTINGS = { 'minimize_on_close': True, 'send_chatstate': HAS_ACCOUNT_DEFAULT, 'encryption': '', + 'sync_threshold': HAS_APP_DEFAULT, }, } @@ -449,8 +449,6 @@ ADVANCED_SETTINGS = { 'ignore_incoming_attention': _('If enabled, Gajim will ignore incoming attention requests (\'wizz\').'), 'remember_opened_chat_controls': _('If enabled, Gajim will reopen chat windows that were opened last time Gajim was closed.'), 'remote_commands': _('If enabled, Gajim will execute XEP-0146 Commands.'), - 'threshold_options': _('Options in days which can be chosen in the sync threshold menu'), - 'private_room_sync_threshold': _('Maximum history in days we request from a private group chat archive. 0: As much as possible.'), 'muclumbus_api_jid': '', 'muclumbus_api_http_uri': '', 'muclumbus_api_pref': _('API Preferences. Possible values: \'http\', \'iq\''), ===================================== gajim/common/storage/archive.py ===================================== @@ -41,7 +41,7 @@ from gajim.common.storage.base import SqliteStorage from gajim.common.storage.base import timeit -CURRENT_USER_VERSION = 5 +CURRENT_USER_VERSION = 6 ARCHIVE_SQL_STATEMENT = ''' CREATE TABLE jids( @@ -78,7 +78,6 @@ ARCHIVE_SQL_STATEMENT = ''' last_mam_id TEXT, oldest_mam_timestamp TEXT, last_muc_timestamp TEXT, - sync_threshold INTEGER ); CREATE INDEX idx_logs_jid_id_time ON logs (jid_id, time DESC); CREATE INDEX idx_logs_stanza_id ON logs (stanza_id); @@ -940,14 +939,13 @@ class MessageArchiveStorage(SqliteStorage): if not exists: sql = '''INSERT INTO last_archive_message (jid_id, last_mam_id, oldest_mam_timestamp, - last_muc_timestamp, sync_threshold) - VALUES (?, ?, ?, ?, ?)''' + last_muc_timestamp) + VALUES (?, ?, ?, ?)''' self._con.execute(sql, ( jid_id, kwargs.get('last_mam_id', None), kwargs.get('oldest_mam_timestamp', None), kwargs.get('last_muc_timestamp', None), - kwargs.get('sync_threshold', None) )) else: for key, value in list(kwargs.items()): ===================================== gajim/groupchat_control.py ===================================== @@ -268,6 +268,13 @@ class GroupchatControl(ChatControlBase): def disco_info(self): return app.storage.cache.get_last_disco_info(self.contact.jid) + @property + def context(self): + disco_info = self.disco_info + if disco_info is None or not self.disco_info.muc_is_members_only: + return 'public' + return 'private' + def add_actions(self): super().add_actions() actions = [ @@ -322,12 +329,8 @@ class GroupchatControl(ChatControlBase): act.connect('change-state', self._on_send_chatstate) self.parent_win.window.add_action(act) - context = 'public' - if self.disco_info is not None and self.disco_info.muc_is_members_only: - context = 'private' - value = self.contact.settings.get('notify_on_all_messages', - context=context) + context=self.context) act = Gio.SimpleAction.new_stateful( 'notify-on-message-' + self.control_id, @@ -351,9 +354,8 @@ class GroupchatControl(ChatControlBase): act.connect('change-state', self._on_print_join_left) self.parent_win.window.add_action(act) - archive_info = app.storage.archive.get_archive_infos(self.contact.jid) - threshold = helpers.get_sync_threshold(self.contact.jid, - archive_info) + threshold = self.contact.settings.get('sync_threshold', + context=self.context) initial = GLib.Variant.new_string(str(threshold)) act = Gio.SimpleAction.new_stateful( @@ -534,9 +536,8 @@ class GroupchatControl(ChatControlBase): # After the room has been created, reevaluate threshold if self.disco_info.has_mam: - archive_info = app.storage.archive.get_archive_infos(self.contact.jid) - threshold = helpers.get_sync_threshold(self.contact.jid, - archive_info) + threshold = self.contact.settings.get('sync_threshold', + context=self.context) win.change_action_state('choose-sync-%s' % self.control_id, GLib.Variant('s', str(threshold))) @@ -661,7 +662,9 @@ class GroupchatControl(ChatControlBase): def _on_sync_threshold(self, action, param): threshold = param.get_string() action.set_state(param) - app.storage.archive.set_archive_infos(self.contact.jid, sync_threshold=threshold) + self.contact.settings.set('sync_threshold', + int(threshold), + context=self.context) def _on_execute_command(self, _action, param): jid = self.room_jid ===================================== gajim/gtk/preferences.py ===================================== @@ -21,9 +21,9 @@ from gi.repository import Gdk from gajim.common import app from gajim.common import configpaths from gajim.common import helpers +from gajim.common.const import THRESHOLD_OPTIONS from gajim.common.nec import NetworkEvent from gajim.common.i18n import _ -from gajim.common.i18n import ngettext from gajim.common.helpers import open_file from gajim.common.multimedia_helpers import AudioInputManager from gajim.common.multimedia_helpers import AudioOutputManager @@ -390,16 +390,6 @@ class Chats(PreferenceBox): class GroupChats(PreferenceBox): def __init__(self, *args): - sync_threshold = {} - options = app.settings.get('threshold_options').split(',') - days = [int(option.strip()) for option in options] - for day in days: - if day == 0: - label = _('No threshold') - else: - label = ngettext('%i day', '%i days', day, day, day) - sync_threshold[day] = label - settings = [ Setting(SettingKind.SWITCH, @@ -416,9 +406,9 @@ class GroupChats(PreferenceBox): Setting(SettingKind.POPOVER, _('Default Sync Threshold'), SettingType.CONFIG, - 'public_room_sync_threshold', + 'gc_sync_threshold_public_default', desc=_('Default for new public group chats'), - props={'entries': sync_threshold}, + props={'entries': THRESHOLD_OPTIONS}, callback=self._on_sync_threshold_changed), Setting(SettingKind.SWITCH, ===================================== gajim/gui_menu_builder.py ===================================== @@ -22,10 +22,10 @@ from gajim.common import app from gajim.common import helpers from gajim.common.helpers import is_affiliation_change_allowed from gajim.common.helpers import is_role_change_allowed -from gajim.common.i18n import ngettext from gajim.common.i18n import _ from gajim.common.const import URIType from gajim.common.const import URIAction +from gajim.common.const import THRESHOLD_OPTIONS from gajim.gtk.util import get_builder from gajim.gtk.const import ControlType @@ -633,15 +633,9 @@ def get_groupchat_menu(control_id, account, jid): def build_sync_menu(): menu = Gio.Menu() - days = app.settings.get('threshold_options').split(',') - days = [int(day) for day in days] action_name = 'win.choose-sync-%s::' % control_id - for day in days: - if day == 0: - label = _('No threshold') - else: - label = ngettext('%i day', '%i days', day, day, day) - menu.append(label, '%s%s' % (action_name, day)) + for days, label in THRESHOLD_OPTIONS.items(): + menu.append(label, '%s%s' % (action_name, days)) return menu def build_chatstate_menu(): View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/a72de0396002a6d28ea25a22a4c8fc61… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/a72de0396002a6d28ea25a22a4c8fc61… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] 2 commits: Flatpak: Fix support for plugins with complex dependencies
by André 18 Sep '20

18 Sep '20

André pushed to branch master at gajim / gajim Commits: 75dc9c81 by André Apitzsch at 2020-09-18T22:39:07+02:00 Flatpak: Fix support for plugins with complex dependencies Move site-packages next to lib directory because merge dirs doesn't work with subdirectories. - - - - - b72d0a6e by André Apitzsch at 2020-09-18T22:39:46+02:00 Flatpak: Update dependencies - - - - - 1 changed file: - flatpak/org.gajim.Gajim.yaml Changes: ===================================== flatpak/org.gajim.Gajim.yaml ===================================== @@ -30,13 +30,13 @@ finish-args: # camera access - --device=all # extensions - - --env=PYTHONPATH=/app/plugins/lib/python3.7/site-packages + - --env=PYTHONPATH=/app/plugins/site-packages - --env=GI_TYPELIB_PATH=/app/lib/girepository-1.0:/app/plugins/lib/girepository-1.0 add-extensions: org.gajim.Gajim.Devel.Plugin: directory: plugins - merge-dirs: lib + merge-dirs: lib;site-packages add-ld-path: lib subdirectories: true no-autodownload: true @@ -112,8 +112,8 @@ modules: - python3 setup.py install --prefix=/app sources: - type: archive - url: https://files.pythonhosted.org/packages/bf/ac/552fc8729d90393845cc3a2062fac… - sha256: 8e924dbc025206e97756e8903039662aa58aa9ba357d8e1d8fc29e3092322053 + url: https://files.pythonhosted.org/packages/12/be/c9cc7d7ab71dbcc9e4e517ead0cdd… + sha256: 26409a473cc6278e4c90f782cd5968ebad04d3911ed1c402fc86908c17633e08 - name: python3-pyopenssl buildsystem: simple @@ -172,13 +172,13 @@ modules: - name: python3-keyring buildsystem: simple build-commands: - - pip3 install --prefix=/app keyring-21.3.0-py3-none-any.whl + - pip3 install --prefix=/app keyring-21.4.0-py3-none-any.whl cleanup: - /bin sources: - type: file - url: https://files.pythonhosted.org/packages/59/4d/469272124b46456c425a505a8b5f2… - sha256: e7a17caf40c40b6bb8c4772224a487e4a63013560ed0c521065aeba7ecd42182 + url: https://files.pythonhosted.org/packages/e4/ed/7be20815f248b0d6aae406783c2be… + sha256: 4e34ea2fdec90c1c43d6610b5a5fafa1b9097db1802948e90caf5763974b8f8d - name: python3-css-parser buildsystem: simple @@ -210,8 +210,8 @@ modules: - /bin sources: - type: archive - url: https://download.gnome.org/sources/gspell/1.8/gspell-1.8.3.tar.xz - sha256: 5ae514dd0216be069176accf6d0049d6a01cfa6a50df4bc06be85f7080b62de8 + url: https://download.gnome.org/sources/gspell/1.8/gspell-1.8.4.tar.xz + sha256: cf4d16a716e813449bd631405dc1001ea89537b8cdae2b8abfb3999212bd43b4 - name: farstream sources: View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/7703d40e2aa52f37ff2fc173a466e27… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/7703d40e2aa52f37ff2fc173a466e27… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] ChatControlBase: Don’t fail on empty clipboard
by Philipp Hörist 18 Sep '20

18 Sep '20

Philipp Hörist pushed to branch master at gajim / gajim Commits: 7703d40e by lovetox at 2020-09-18T10:24:13+02:00 ChatControlBase: Don’t fail on empty clipboard Fixes #10272 - - - - - 1 changed file: - gajim/chat_control_base.py Changes: ===================================== gajim/chat_control_base.py ===================================== @@ -729,6 +729,8 @@ class ChatControlBase(ChatCommandProcessor, CommandTools, EventHelper): def paste_clipboard_as_quote(self, _item: Gtk.MenuItem) -> None: clipboard = Gtk.Clipboard.get(Gdk.SELECTION_CLIPBOARD) text = clipboard.wait_for_text() + if text is None: + return self.insert_as_quote(text) def on_quote(self, _widget, text): View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/7703d40e2aa52f37ff2fc173a466e271… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/7703d40e2aa52f37ff2fc173a466e271… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] Settings: Init in Application startup
by Philipp Hörist 18 Sep '20

18 Sep '20

Philipp Hörist pushed to branch master at gajim / gajim Commits: deb72c03 by lovetox at 2020-09-18T10:22:50+02:00 Settings: Init in Application startup - - - - - 4 changed files: - gajim/application.py - gajim/common/settings.py - gajim/common/types.py - gajim/gui_interface.py Changes: ===================================== gajim/application.py ===================================== @@ -57,6 +57,8 @@ from gajim.common.contacts import LegacyContactsAPI from gajim.common.task_manager import TaskManager from gajim.common.storage.cache import CacheStorage from gajim.common.storage.archive import MessageArchiveStorage +from gajim.common.settings import Settings +from gajim.common.settings import LegacyConfig class GajimApplication(Gtk.Application): @@ -184,6 +186,11 @@ class GajimApplication(Gtk.Application): app.detect_dependencies() configpaths.create_paths() + app.settings = Settings() + app.settings.init() + + app.config = LegacyConfig() # type: ignore + app.storage.cache = CacheStorage() app.storage.cache.init() ===================================== gajim/common/settings.py ===================================== @@ -70,7 +70,7 @@ CREATE_SQL = ''' json.dumps(PROXY_EXAMPLES)) -class _Settings: +class Settings: def __init__(self): self._con = None self._commit_scheduled = None @@ -158,8 +158,6 @@ class _Settings: self._migrate_old_config() self._commit() self._migrate_database() - app.settings = self # type: ignore - app.config = LegacyConfig() # type: ignore @staticmethod def _setup_installation_defaults() -> None: @@ -890,30 +888,27 @@ class _Settings: self.set_account_setting(account, 'proxy', None) -Settings = _Settings() - - class LegacyConfig: @staticmethod def get(setting: str) -> SETTING_TYPE: - return Settings.get_app_setting(setting) + return app.settings.get_app_setting(setting) @staticmethod def set(setting: str, value: SETTING_TYPE) -> None: - Settings.set_app_setting(setting, value) + app.settings.set_app_setting(setting, value) @staticmethod def get_per(kind: str, key: str, setting: str) -> SETTING_TYPE: if kind == 'accounts': - return Settings.get_account_setting(key, setting) + return app.settings.get_account_setting(key, setting) if kind == 'plugins': - return Settings.get_plugin_setting(key, setting) + return app.settings.get_plugin_setting(key, setting) raise ValueError @staticmethod def set_per(kind: str, key: str, setting: str, value: SETTING_TYPE) -> None: if kind == 'accounts': - Settings.set_account_setting(key, setting, value) + app.settings.set_account_setting(key, setting, value) raise ValueError ===================================== gajim/common/types.py ===================================== @@ -37,7 +37,7 @@ if TYPE_CHECKING: from gajim.common.nec import NetworkEventsController from gajim.gui_interface import Interface - from gajim.common.settings import _Settings + from gajim.common.settings import Settings NetworkEventsControllerT = Union['NetworkEventsController'] @@ -61,4 +61,4 @@ PluginExtensionPoints = Dict[str, Tuple[Optional[Callable[..., None]], EventHandlersDict = Dict[str, Tuple[int, Callable[['NetworkEvent'], Optional[bool]]]] PluginEvents = List['NetworkEvent'] -SettingsT = Union['_Settings'] +SettingsT = Union['Settings'] ===================================== gajim/gui_interface.py ===================================== @@ -81,7 +81,6 @@ from gajim.common.structs import MUCData from gajim.common.nec import NetworkEvent from gajim.common.i18n import _ from gajim.common.client import Client -from gajim.common.settings import Settings from gajim.common.const import Display from gajim.common.file_props import FilesProp @@ -2023,8 +2022,6 @@ class Interface: self.avatar_storage = AvatarStorage() - Settings.init() - # Load CSS files app.load_css_config() View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/deb72c03565d921e0508623f5f0a5c2c… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/deb72c03565d921e0508623f5f0a5c2c… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] 2 commits: Refactor message archive storage
by Philipp Hörist 17 Sep '20

17 Sep '20

Philipp Hörist pushed to branch master at gajim / gajim Commits: 71f59c47 by lovetox at 2020-09-17T23:11:09+02:00 Refactor message archive storage - Rename Logger to MessageArchiveStorage - Move into storage module - Inherit from SqliteStorage - - - - - 107417f7 by lovetox at 2020-09-17T23:15:21+02:00 Update pylintrc - - - - - 22 changed files: - gajim/application.py - gajim/chat_control.py - gajim/command_system/implementation/standard.py - gajim/common/app.py - gajim/common/helpers.py - gajim/common/modules/mam.py - gajim/common/modules/message.py - gajim/common/modules/muc.py - gajim/common/modules/presence.py - gajim/common/modules/receipts.py - gajim/common/logger.py → gajim/common/storage/archive.py - gajim/common/storage/base.py - gajim/common/types.py - gajim/common/zeroconf/connection_handlers_zeroconf.py - gajim/groupchat_control.py - gajim/gtk/history.py - gajim/gtk/history_sync.py - gajim/gui_interface.py - gajim/roster_window.py - gajim/session.py - pylintrc - test/broken/unit/test_sessions.py Changes: ===================================== gajim/application.py ===================================== @@ -52,11 +52,11 @@ from gajim.common import ged from gajim.common import configpaths from gajim.common import logging_helpers from gajim.common import exceptions -from gajim.common import logger from gajim.common.i18n import _ from gajim.common.contacts import LegacyContactsAPI from gajim.common.task_manager import TaskManager from gajim.common.storage.cache import CacheStorage +from gajim.common.storage.archive import MessageArchiveStorage class GajimApplication(Gtk.Application): @@ -183,10 +183,14 @@ class GajimApplication(Gtk.Application): app.print_version() app.detect_dependencies() configpaths.create_paths() + app.storage.cache = CacheStorage() app.storage.cache.init() + + app.storage.archive = MessageArchiveStorage() + app.storage.archive.init() + try: - app.logger = logger.Logger() app.contacts = LegacyContactsAPI() except exceptions.DatabaseMalformed as error: dlg = Gtk.MessageDialog( @@ -296,7 +300,8 @@ class GajimApplication(Gtk.Application): self.interface.roster.prepare_quit() # Commit any outstanding SQL transactions - app.logger.commit() + app.storage.cache.shutdown() + app.storage.archive.shutdown() def _handle_remote_options(self, _application, command_line): # Parse all options that should be executed on a remote instance ===================================== gajim/chat_control.py ===================================== @@ -637,7 +637,7 @@ class ChatControl(ChatControlBase): end = self.conv_textview.autoscroll if not pw or (pw.get_active_control() and self \ == pw.get_active_control() and pw.is_active() and end): - app.logger.set_read_messages([event.msg_log_id]) + app.storage.archive.set_read_messages([event.msg_log_id]) @event_filter(['account', 'jid']) def _on_message_error(self, event): @@ -1221,7 +1221,7 @@ class ChatControl(ChatControlBase): self.contact.get_full_jid(), ['chat', 'pm'])) - rows = app.logger.get_last_conversation_lines( + rows = app.storage.archive.get_last_conversation_lines( self.account, jid, pending) local_old_kind = None @@ -1299,7 +1299,7 @@ class ChatControl(ChatControlBase): if event.session and not self.session: self.set_session(event.session) if message_ids: - app.logger.set_read_messages(message_ids) + app.storage.archive.set_read_messages(message_ids) app.events.remove_events(self.account, jid_with_resource, types=[str(self._type)]) ===================================== gajim/command_system/implementation/standard.py ===================================== @@ -90,7 +90,7 @@ class StandardCommonCommands(CommandContainer): @command('lastlog', overlap=True) @doc(_("Show logged messages which mention given text")) def grep(self, text, limit=None): - results = app.logger.search_log(self.account, self.contact.jid, text) + results = app.storage.archive.search_log(self.account, self.contact.jid, text) if not results: raise CommandError(_("%s: Nothing found") % text) ===================================== gajim/common/app.py ===================================== @@ -49,7 +49,6 @@ from gajim.common.const import Display from gajim.common.events import Events from gajim.common.types import NetworkEventsControllerT # pylint: disable=unused-import from gajim.common.types import InterfaceT # pylint: disable=unused-import -from gajim.common.types import LoggerT # pylint: disable=unused-import from gajim.common.types import ConnectionT # pylint: disable=unused-import from gajim.common.types import LegacyContactsAPIT # pylint: disable=unused-import from gajim.common.types import SettingsT # pylint: disable=unused-import @@ -69,11 +68,10 @@ ged = ged_module.GlobalEventsDispatcher() # Global Events Dispatcher nec = cast(NetworkEventsControllerT, None) plugin_manager = None # Plugins Manager -logger = cast(LoggerT, None) - class Storage: def __init__(self): self.cache = None + self.archive = None storage = Storage() ===================================== gajim/common/helpers.py ===================================== @@ -904,7 +904,7 @@ def get_sync_threshold(jid, archive_info): threshold = app.settings.get('private_room_sync_threshold') else: threshold = app.settings.get('public_room_sync_threshold') - app.logger.set_archive_infos(jid, sync_threshold=threshold) + app.storage.archive.set_archive_infos(jid, sync_threshold=threshold) return threshold return archive_info.sync_threshold ===================================== gajim/common/modules/mam.py ===================================== @@ -148,9 +148,10 @@ class MAM(BaseModule): if not self.is_catch_up_finished(archive_jid): return - app.logger.set_archive_infos(archive_jid, - last_mam_id=properties.stanza_id.id, - last_muc_timestamp=timestamp) + app.storage.archive.set_archive_infos( + archive_jid, + last_mam_id=properties.stanza_id.id, + last_muc_timestamp=timestamp) def _mam_message_received(self, _con, stanza, properties): if not properties.is_mam_message: @@ -187,11 +188,11 @@ class MAM(BaseModule): stanza_id, message_id = self._get_unique_id(properties) # Search for duplicates - if app.logger.find_stanza_id(self._account, - str(properties.mam.archive), - stanza_id, - message_id, - groupchat=is_groupchat): + if app.storage.archive.find_stanza_id(self._account, + str(properties.mam.archive), + stanza_id, + message_id, + groupchat=is_groupchat): self._log.info('Found duplicate with stanza-id: %s, ' 'message-id: %s', stanza_id, message_id) raise nbxmpp.NodeProcessed @@ -229,16 +230,17 @@ class MAM(BaseModule): return stanza_id = message_id - app.logger.insert_into_logs(self._account, - with_, - properties.mam.timestamp, - kind, - unread=False, - message=msgtxt, - contact_name=properties.muc_nickname, - additional_data=additional_data, - stanza_id=stanza_id, - message_id=properties.id) + app.storage.archive.insert_into_logs( + self._account, + with_, + properties.mam.timestamp, + kind, + unread=False, + message=msgtxt, + contact_name=properties.muc_nickname, + additional_data=additional_data, + stanza_id=stanza_id, + message_id=properties.id) app.nec.push_incoming_event( NetworkEvent('mam-decrypted-message-received', @@ -268,7 +270,7 @@ class MAM(BaseModule): self._log.warning('Request already running: %s', own_jid) return - archive = app.logger.get_archive_infos(own_jid) + archive = app.storage.archive.get_archive_infos(own_jid) mam_id = None if archive is not None: @@ -299,7 +301,7 @@ class MAM(BaseModule): self._catch_up_finished.remove(own_jid) def request_archive_on_muc_join(self, jid): - archive = app.logger.get_archive_infos(jid) + archive = app.storage.archive.get_archive_infos(jid) threshold = get_sync_threshold(jid, archive) self._log.info('Threshold for %s: %s', jid, threshold) @@ -359,7 +361,7 @@ class MAM(BaseModule): if is_error_result(result): if result.condition == 'item-not-found': - app.logger.reset_archive_infos(result.jid) + app.storage.archive.reset_archive_infos(result.jid) if groupchat: self.request_archive_on_muc_join(result.jid) else: @@ -375,21 +377,22 @@ class MAM(BaseModule): # <last> is not provided if the requested page was empty # so this means we did not get anything hence we only need # to update the archive info if <last> is present - app.logger.set_archive_infos(result.jid, - last_mam_id=result.rsm.last, - last_muc_timestamp=time.time()) + app.storage.archive.set_archive_infos( + result.jid, + last_mam_id=result.rsm.last, + last_muc_timestamp=time.time()) if start_date is not None and not groupchat: # Record the earliest timestamp we request from # the account archive. For the account archive we only # set start_date at the very first request. - app.logger.set_archive_infos( + app.storage.archive.set_archive_infos( result.jid, oldest_mam_timestamp=start_date.timestamp()) else: - app.logger.set_archive_infos(result.jid, - last_mam_id=result.rsm.last) + app.storage.archive.set_archive_infos(result.jid, + last_mam_id=result.rsm.last) queryid = self._get_query_id(result.jid) self._nbxmpp('MAM').make_query(result.jid, @@ -445,8 +448,8 @@ class MAM(BaseModule): if result.complete: self._log.info('Request finished: %s, last mam id: %s', result.jid, result.rsm.last) - app.logger.set_archive_infos(result.jid, - oldest_mam_timestamp=timestamp) + app.storage.archive.set_archive_infos( + result.jid, oldest_mam_timestamp=timestamp) app.nec.push_incoming_event(NetworkEvent( 'archiving-interval-finished', account=self._account, ===================================== gajim/common/modules/message.py ===================================== @@ -114,20 +114,22 @@ class Message(BaseModule): # Only for XEP-0045 MUC History # Don’t check for message text because the message could be # encrypted. - if app.logger.deduplicate_muc_message(self._account, - properties.jid.bare, - properties.jid.resource, - properties.timestamp, - properties.id): + if app.storage.archive.deduplicate_muc_message( + self._account, + properties.jid.bare, + properties.jid.resource, + properties.timestamp, + properties.id): raise nbxmpp.NodeProcessed if (properties.is_self_message or properties.is_muc_pm): archive_jid = self._con.get_own_jid().bare - if app.logger.find_stanza_id(self._account, - archive_jid, - stanza_id, - message_id, - properties.type.is_groupchat): + if app.storage.archive.find_stanza_id( + self._account, + archive_jid, + stanza_id, + message_id, + properties.type.is_groupchat): return msgtxt = properties.body @@ -220,10 +222,11 @@ class Message(BaseModule): self._log.info(properties.error) - app.logger.set_message_error(app.get_jid_from_account(self._account), - jid, - properties.id, - properties.error) + app.storage.archive.set_message_error( + app.get_jid_from_account(self._account), + jid, + properties.id, + properties.error) app.nec.push_incoming_event( NetworkEvent('message-error', @@ -241,7 +244,7 @@ class Message(BaseModule): # if not event.nick, it means message comes from room itself # usually it hold description and can be send at each connection # so don't store it in logs - app.logger.insert_into_logs( + app.storage.archive.insert_into_logs( self._account, event.jid, event.properties.timestamp, @@ -361,15 +364,16 @@ class Message(BaseModule): if message.message is None: return - app.logger.insert_into_logs(self._account, - message.jid, - message.timestamp, - message.kind, - message=message.message, - subject=message.subject, - additional_data=message.additional_data, - message_id=message.message_id, - stanza_id=message.message_id) + app.storage.archive.insert_into_logs( + self._account, + message.jid, + message.timestamp, + message.kind, + message=message.message, + subject=message.subject, + additional_data=message.additional_data, + message_id=message.message_id, + stanza_id=message.message_id) def get_instance(*args, **kwargs): ===================================== gajim/common/modules/muc.py ===================================== @@ -555,9 +555,9 @@ class MUC(BaseModule): show = 'offline' else: show = properties.show.value - show = app.logger.convert_show_values_to_db_api_values(show) + show = app.storage.archive.convert_show_values_to_db_api_values(show) - app.logger.insert_into_logs( + app.storage.archive.insert_into_logs( self._account, properties.jid.bare, properties.timestamp, ===================================== gajim/common/modules/presence.py ===================================== @@ -236,12 +236,12 @@ class Presence(BaseModule): if properties.type.is_unavailable: show = ShowConstant.OFFLINE - app.logger.insert_into_logs(self._account, - properties.jid.bare, - time.time(), - KindConstant.STATUS, - message=properties.status, - show=show) + app.storage.archive.insert_into_logs(self._account, + properties.jid.bare, + time.time(), + KindConstant.STATUS, + message=properties.status, + show=show) def _subscribe_received(self, _con, _stanza, properties): jid = properties.jid.bare ===================================== gajim/common/modules/receipts.py ===================================== @@ -81,10 +81,11 @@ class Receipts(BaseModule): if not properties.is_muc_pm: jid = jid.new_as_bare() - app.logger.set_marker(app.get_jid_from_account(self._account), - jid, - properties.receipt.id, - 'received') + app.storage.archive.set_marker( + app.get_jid_from_account(self._account), + jid, + properties.receipt.id, + 'received') app.nec.push_incoming_event( NetworkEvent('receipt-received', ===================================== gajim/common/logger.py → gajim/common/storage/archive.py ===================================== @@ -21,35 +21,29 @@ # You should have received a copy of the GNU General Public License # along with Gajim. If not, see <http://www.gnu.org/licenses/>. -""" -This module allows to access the on-disk database of logs -""" - -import os -import sys import time import datetime import calendar import json import logging import sqlite3 as sqlite +from pathlib import Path from collections import namedtuple -from gi.repository import GLib - -from gajim.common import exceptions from gajim.common import app from gajim.common import configpaths from gajim.common.helpers import AdditionalDataDict -from gajim.common.i18n import _ from gajim.common.const import ShowConstant from gajim.common.const import KindConstant from gajim.common.const import JIDConstant +from gajim.common.storage.base import SqliteStorage from gajim.common.storage.base import timeit -LOGS_SQL_STATEMENT = ''' +CURRENT_USER_VERSION = 5 + +ARCHIVE_SQL_STATEMENT = ''' CREATE TABLE jids( jid_id INTEGER PRIMARY KEY AUTOINCREMENT UNIQUE, jid TEXT UNIQUE, @@ -89,76 +83,57 @@ LOGS_SQL_STATEMENT = ''' CREATE INDEX idx_logs_jid_id_time ON logs (jid_id, time DESC); CREATE INDEX idx_logs_stanza_id ON logs (stanza_id); CREATE INDEX idx_logs_message_id ON logs (message_id); - PRAGMA user_version=5; - ''' + PRAGMA user_version=%s; + ''' % CURRENT_USER_VERSION -log = logging.getLogger('gajim.c.logger') +log = logging.getLogger('gajim.c.storage.archive') -class Logger: +class MessageArchiveStorage(SqliteStorage): def __init__(self): - self._log = log + SqliteStorage.__init__(self, + log, + Path(configpaths.get('LOG_DB')), + ARCHIVE_SQL_STATEMENT) + self._jid_ids = {} self._jid_ids_reversed = {} - self._con = None - self._commit_timout_id = None - self._log_db_path = configpaths.get('LOG_DB') - - self._create_databases() - self._migrate_databases() - self._connect_databases() - self._get_jid_ids_from_db() - - def _create_databases(self): - if os.path.isdir(self._log_db_path): - log.error(_('%s is a directory but should be a file'), - self._log_db_path) - sys.exit() - if not os.path.exists(self._log_db_path): - self._create(LOGS_SQL_STATEMENT, self._log_db_path) + def init(self, **kwargs): + SqliteStorage.init(self, + detect_types=sqlite.PARSE_COLNAMES) - @staticmethod - def _connect(*args, **kwargs): - con = sqlite.connect(*args, **kwargs) - con.execute("PRAGMA secure_delete=1") - return con + self._set_journal_mode('WAL') + self._enable_secure_delete() - @classmethod - def _create(cls, statement, path): - log.info(_('Creating %s'), path) - con = cls._connect(path) - os.chmod(path, 0o600) + self._con.row_factory = self._namedtuple_factory - try: - con.executescript(statement) - except Exception: - log.exception('Error') - con.close() - os.remove(path) - sys.exit() + self._con.create_function("like", 1, self._like) + self._con.create_function("get_timeout", 0, self._get_timeout) - con.commit() - con.close() + self._get_jid_ids_from_db() - @staticmethod - def _get_user_version(con: sqlite.Connection) -> int: - """ Return the value of PRAGMA user_version. """ - return con.execute('PRAGMA user_version').fetchone()[0] + def _namedtuple_factory(self, cursor, row): + fields = [col[0] for col in cursor.description] + Row = namedtuple("Row", fields) + named_row = Row(*row) + if 'additional_data' in fields: + _dict = json.loads(named_row.additional_data or '{}') + named_row = named_row._replace( + additional_data=AdditionalDataDict(_dict)) - def _migrate_databases(self): - try: - con = self._connect(self._log_db_path) - self._migrate_logs(con) - con.close() - except Exception: - log.exception('Error') - sys.exit() + # if an alias `account` for the field `account_id` is used for the + # query, the account_id is converted to the account jid + if 'account' in fields: + if named_row.account: + jid = self._jid_ids_reversed[named_row.account].jid + named_row = named_row._replace(account=jid) + return named_row - @timeit - def _migrate_logs(self, con): - if self._get_user_version(con) == 0: + def _migrate(self): + user_version = self.user_version + if user_version == 0: # All migrations from 0.16.9 until 1.0.0 statements = [ 'ALTER TABLE logs ADD COLUMN "account_id" INTEGER', @@ -179,101 +154,40 @@ class Logger: 'PRAGMA user_version=1' ] - self._execute_multiple(con, statements) + self._execute_multiple(statements) - if self._get_user_version(con) < 2: + if user_version < 2: statements = [ 'ALTER TABLE last_archive_message ADD COLUMN "sync_threshold" INTEGER', 'PRAGMA user_version=2' ] - self._execute_multiple(con, statements) + self._execute_multiple(statements) - if self._get_user_version(con) < 3: + if user_version < 3: statements = [ 'ALTER TABLE logs ADD COLUMN "message_id" TEXT', 'PRAGMA user_version=3' ] - self._execute_multiple(con, statements) + self._execute_multiple(statements) - if self._get_user_version(con) < 4: + if user_version < 4: statements = [ 'ALTER TABLE logs ADD COLUMN "error" TEXT', 'PRAGMA user_version=4' ] - self._execute_multiple(con, statements) + self._execute_multiple(statements) - if self._get_user_version(con) < 5: + if user_version < 5: statements = [ 'CREATE INDEX idx_logs_message_id ON logs (message_id)', 'PRAGMA user_version=5' ] - self._execute_multiple(con, statements) + self._execute_multiple(statements) @staticmethod - def _execute_multiple(con, statements): - """ - Execute multiple statements with the option to fail on duplicates - but still continue - """ - for sql in statements: - try: - con.execute(sql) - con.commit() - except sqlite.OperationalError as error: - if str(error).startswith('duplicate column name:'): - log.info(error) - else: - log.exception('Error') - sys.exit() - - def namedtuple_factory(self, cursor, row): - """ - Usage: - con.row_factory = namedtuple_factory - """ - fields = [col[0] for col in cursor.description] - Row = namedtuple("Row", fields) - named_row = Row(*row) - if 'additional_data' in fields: - _dict = json.loads(named_row.additional_data or '{}') - named_row = named_row._replace( - additional_data=AdditionalDataDict(_dict)) - - # if an alias `account` for the field `account_id` is used for the - # query, the account_id is converted to the account jid - if 'account' in fields: - if named_row.account: - jid = self._jid_ids_reversed[named_row.account].jid - named_row = named_row._replace(account=jid) - return named_row - - def dispatch(self, event, error): + def dispatch(event, error): app.ged.raise_event(event, None, str(error)) - def _connect_databases(self): - self._con = self._connect(self._log_db_path, - timeout=20.0, - isolation_level='IMMEDIATE', - detect_types=sqlite.PARSE_COLNAMES) - - self._con.row_factory = self.namedtuple_factory - - # DB functions - self._con.create_function("like", 1, self._like) - self._con.create_function("get_timeout", 0, self._get_timeout) - - self._set_synchronous(False) - - @timeit - def _set_synchronous(self, sync): - try: - if sync: - self._con.execute("PRAGMA synchronous = NORMAL") - else: - self._con.execute("PRAGMA synchronous = OFF") - except sqlite.Error: - log.exception('Error') - @staticmethod def _get_timeout(): """ @@ -290,28 +204,6 @@ class Logger: def _like(search_str): return '%{}%'.format(search_str) - @timeit - def commit(self): - try: - self._con.commit() - except sqlite.OperationalError as e: - print(str(e), file=sys.stderr) - self._commit_timout_id = None - return False - - def _timeout_commit(self): - if self._commit_timout_id: - return - self._commit_timout_id = GLib.timeout_add(500, self.commit) - - @timeit - def simple_commit(self, sql_to_commit): - """ - Helper to commit - """ - self._con.execute(sql_to_commit) - self._timeout_commit() - @timeit def _get_jid_ids_from_db(self): """ @@ -347,7 +239,7 @@ class Logger: """ jid_ = self._jid_ids.get(jid) if jid_ is None: - return + return None return jid_.type == JIDConstant.ROOM_TYPE @staticmethod @@ -408,10 +300,11 @@ class Logger: lastrowid = self._con.execute(sql, (jid, type_)).lastrowid Row = namedtuple('Row', 'jid_id jid type') self._jid_ids[jid] = Row(lastrowid, jid, type_) - self._timeout_commit() + self._delayed_commit() return lastrowid - def convert_show_values_to_db_api_values(self, show): + @staticmethod + def convert_show_values_to_db_api_values(show): """ Convert from string style to constant ints for db """ @@ -442,7 +335,7 @@ class Logger: sql = '''INSERT INTO unread_messages (message_id, jid_id, shown) VALUES (?, ?, 0)''' self._con.execute(sql, (message_id, jid_id)) - self._timeout_commit() + self._delayed_commit() @timeit def set_read_messages(self, message_ids): @@ -451,7 +344,8 @@ class Logger: """ ids = ','.join([str(i) for i in message_ids]) sql = 'DELETE FROM unread_messages WHERE message_id IN (%s)' % ids - self.simple_commit(sql) + self._con.execute(sql) + self._delayed_commit() @timeit def set_shown_unread_msgs(self, msg_log_id): @@ -460,7 +354,8 @@ class Logger: """ sql = 'UPDATE unread_messages SET shown = 1 where message_id = %s' % \ msg_log_id - self.simple_commit(sql) + self._con.execute(sql) + self._delayed_commit() @timeit def reset_shown_unread_messages(self): @@ -468,7 +363,8 @@ class Logger: Set shown field to False in unread_messages table """ sql = 'UPDATE unread_messages SET shown = 0' - self.simple_commit(sql) + self._con.execute(sql) + self._delayed_commit() @timeit def get_unread_msgs(self): @@ -495,13 +391,15 @@ class Logger: ''' % msg_log_id ).fetchone() if result is None: - # Log line is no more in logs table. remove it from unread_messages + # Log line is no more in logs table. + # remove it from unread_messages self.set_read_messages([msg_log_id]) continue all_messages.append((result, shown)) return all_messages + @timeit def load_groupchat_messages(self, account, jid): account_id = self.get_account_id(account, type_=JIDConstant.ROOM_TYPE) @@ -511,13 +409,8 @@ class Logger: AND account_id = ? AND kind = ? ORDER BY time DESC, log_line_id DESC LIMIT ?''' - try: - messages = self._con.execute( - sql, (jid, account_id, KindConstant.GC_MSG, 50)).fetchall() - except sqlite.DatabaseError: - self.dispatch('DB_ERROR', - exceptions.DatabaseMalformed(self._log_db_path)) - return [] + messages = self._con.execute( + sql, (jid, account_id, KindConstant.GC_MSG, 50)).fetchall() messages.reverse() return messages @@ -568,27 +461,12 @@ class Logger: account_id=account_id, kinds=', '.join(kinds)) - try: - messages = self._con.execute( - sql, tuple(jids) + (restore, pending)).fetchall() - except sqlite.DatabaseError: - self.dispatch('DB_ERROR', - exceptions.DatabaseMalformed(self._log_db_path)) - return [] + messages = self._con.execute( + sql, tuple(jids) + (restore, pending)).fetchall() messages.reverse() return messages - def get_unix_time_from_date(self, year, month, day): - # year (fe 2005), month (fe 11), day (fe 25) - # returns time in seconds for the second that starts that date since epoch - # gimme unixtime from year month day: - d = datetime.date(year, month, day) - local_time = d.timetuple() # time tuple (compat with time.localtime()) - # we have time since epoch baby :) - start_of_day = int(time.mktime(local_time)) - return start_of_day - @timeit def get_conversation_for_date(self, account, jid, date): """ @@ -960,7 +838,7 @@ class Logger: VALUES (?, (SELECT jid_id FROM jids WHERE jid = ?))''' self._con.execute(sql, (lastrowid, jid)) - self._timeout_commit() + self._delayed_commit() return lastrowid @@ -991,7 +869,7 @@ class Logger: WHERE account_id = ? AND jid_id = ? AND message_id = ? ''' self._con.execute(sql, (error, account_id, jid_id, message_id)) - self._timeout_commit() + self._delayed_commit() @timeit def set_marker(self, account_jid, jid, message_id, state): @@ -1024,7 +902,7 @@ class Logger: WHERE account_id = ? AND jid_id = ? AND message_id = ? ''' self._con.execute(sql, (state, account_id, jid_id, message_id)) - self._timeout_commit() + self._delayed_commit() @timeit def get_archive_infos(self, jid): @@ -1081,7 +959,7 @@ class Logger: WHERE jid_id = ?'''.format(args) self._con.execute(sql, tuple(kwargs.values()) + (jid_id,)) log.info('Set message archive info: %s %s', jid, kwargs) - self._timeout_commit() + self._delayed_commit() @timeit def reset_archive_infos(self, jid): @@ -1098,4 +976,4 @@ class Logger: WHERE jid_id = ?''' self._con.execute(sql, (jid_id,)) log.info('Reset message archive info: %s', jid) - self._timeout_commit() + self._delayed_commit() ===================================== gajim/common/storage/base.py ===================================== @@ -108,6 +108,9 @@ class SqliteStorage: def _set_synchronous(self, mode): self._con.execute(f'PRAGMA synchronous={mode}') + def _enable_secure_delete(self): + self._con.execute('PRAGMA secure_delete=1') + @property def user_version(self) -> int: return self._con.execute('PRAGMA user_version').fetchone()[0] @@ -177,3 +180,11 @@ class SqliteStorage: self._commit_source_id = GLib.timeout_add(self._commit_delay, self._commit) + + def shutdown(self): + if self._commit_source_id is not None: + GLib.source_remove(self._commit_source_id) + + self._commit() + self._con.close() + self._con = None ===================================== gajim/common/types.py ===================================== @@ -35,7 +35,6 @@ if TYPE_CHECKING: from gajim.common.contacts import LegacyContactsAPI from gajim.common.nec import NetworkEvent from gajim.common.nec import NetworkEventsController - from gajim.common.logger import Logger from gajim.gui_interface import Interface from gajim.common.settings import _Settings @@ -43,7 +42,6 @@ if TYPE_CHECKING: NetworkEventsControllerT = Union['NetworkEventsController'] InterfaceT = Union['Interface'] -LoggerT = Union['Logger'] ConnectionT = Union['Client', 'ConnectionZeroconf'] ContactsT = Union['Contact', 'GC_Contact'] ===================================== gajim/common/zeroconf/connection_handlers_zeroconf.py ===================================== @@ -114,7 +114,7 @@ class ConnectionHandlersZeroconf(connection_handlers.ConnectionHandlersBase): def _message_error_received(self, _con, _stanza, properties): log.info(properties.error) - app.logger.set_message_error(app.get_jid_from_account(self.name), + app.storage.archive.set_message_error(app.get_jid_from_account(self.name), properties.jid, properties.id, properties.error) ===================================== gajim/groupchat_control.py ===================================== @@ -351,7 +351,7 @@ class GroupchatControl(ChatControlBase): act.connect('change-state', self._on_print_join_left) self.parent_win.window.add_action(act) - archive_info = app.logger.get_archive_infos(self.contact.jid) + archive_info = app.storage.archive.get_archive_infos(self.contact.jid) threshold = helpers.get_sync_threshold(self.contact.jid, archive_info) @@ -474,7 +474,7 @@ class GroupchatControl(ChatControlBase): self.parent_win.window.remove_action(f'{action}{self.control_id}') def _restore_conversation(self): - rows = app.logger.load_groupchat_messages( + rows = app.storage.archive.load_groupchat_messages( self.account, self.contact.jid) for row in rows: @@ -534,7 +534,7 @@ class GroupchatControl(ChatControlBase): # After the room has been created, reevaluate threshold if self.disco_info.has_mam: - archive_info = app.logger.get_archive_infos(self.contact.jid) + archive_info = app.storage.archive.get_archive_infos(self.contact.jid) threshold = helpers.get_sync_threshold(self.contact.jid, archive_info) win.change_action_state('choose-sync-%s' % self.control_id, @@ -661,7 +661,7 @@ class GroupchatControl(ChatControlBase): def _on_sync_threshold(self, action, param): threshold = param.get_string() action.set_state(param) - app.logger.set_archive_infos(self.contact.jid, sync_threshold=threshold) + app.storage.archive.set_archive_infos(self.contact.jid, sync_threshold=threshold) def _on_execute_command(self, _action, param): jid = self.room_jid ===================================== gajim/gtk/history.py ===================================== @@ -174,7 +174,7 @@ class HistoryWindow(Gtk.ApplicationWindow): self._ui.query_entry.set_model(liststore) # Add all jids in logs.db: - db_jids = app.logger.get_jids_in_db() + db_jids = app.storage.archive.get_jids_in_db() completion_dict = dict.fromkeys(db_jids) self.accounts_seen_online = list(app.contacts.get_accounts()) @@ -216,10 +216,10 @@ class HistoryWindow(Gtk.ApplicationWindow): info_acc = self._get_account_for_jid(info_jid) - if (app.logger.jid_is_room_jid(completed) or - app.logger.jid_is_from_pm(completed)): + if (app.storage.archive.jid_is_room_jid(completed) or + app.storage.archive.jid_is_from_pm(completed)): icon = muc_active_icon - if app.logger.jid_is_from_pm(completed): + if app.storage.archive.jid_is_from_pm(completed): # It's PM. Make it easier to find room, nick = app.get_room_and_nick_from_fjid(completed) info_completion = '%s from %s' % (nick, room) @@ -336,10 +336,10 @@ class HistoryWindow(Gtk.ApplicationWindow): self.jids_to_search = [info_jid] # Get first/last date we have logs with contact - self.first_log = app.logger.get_first_date_that_has_logs( + self.first_log = app.storage.archive.get_first_date_that_has_logs( self.account, self.jid) self.first_day = self._get_date_from_timestamp(self.first_log) - self.last_log = app.logger.get_last_date_that_has_logs( + self.last_log = app.storage.archive.get_last_date_that_has_logs( self.account, self.jid) self.last_day = self._get_date_from_timestamp(self.last_log) @@ -405,7 +405,7 @@ class HistoryWindow(Gtk.ApplicationWindow): month = python_month(month) try: - log_days = app.logger.get_days_with_logs( + log_days = app.storage.archive.get_days_with_logs( self.account, self.jid, year, month) except exceptions.PysqliteOperationalError as error: ErrorDialog(_('Disk Error'), str(error)) @@ -459,7 +459,7 @@ class HistoryWindow(Gtk.ApplicationWindow): if _date == end_date: break try: - logs = app.logger.get_date_has_logs( + logs = app.storage.archive.get_date_has_logs( self.account, self.jid, _date) except exceptions.PysqliteOperationalError as e: ErrorDialog(_('Disk Error'), str(e)) @@ -497,7 +497,7 @@ class HistoryWindow(Gtk.ApplicationWindow): date = datetime.datetime(year, month, day) - conversation = app.logger.get_conversation_for_date( + conversation = app.storage.archive.get_conversation_for_date( self.account, self.jid, date) for message in conversation: @@ -673,7 +673,7 @@ class HistoryWindow(Gtk.ApplicationWindow): show_status = self._ui.show_status_checkbutton.get_active() - results = app.logger.search_log(account, jid, text, date) + results = app.storage.archive.search_log(account, jid, text, date) result_found = False # FIXME: # add "subject: | message: " in message column if kind is single ===================================== gajim/gtk/history_sync.py ===================================== @@ -64,7 +64,7 @@ class HistorySyncAssistant(Gtk.Assistant, EventHelper): own_jid = self.con.get_own_jid().bare mam_start = ArchiveState.NEVER - archive = app.logger.get_archive_infos(own_jid) + archive = app.storage.archive.get_archive_infos(own_jid) if archive is not None and archive.oldest_mam_timestamp is not None: mam_start = int(float(archive.oldest_mam_timestamp)) ===================================== gajim/gui_interface.py ===================================== @@ -795,7 +795,7 @@ class Interface: # block signed in notifications for 30 seconds # Add our own JID into the DB - app.logger.insert_jid(obj.conn.get_own_jid().bare) + app.storage.archive.insert_jid(obj.conn.get_own_jid().bare) account = obj.conn.name app.block_signed_in_notifications[account] = True @@ -2028,7 +2028,7 @@ class Interface: # Load CSS files app.load_css_config() - app.logger.reset_shown_unread_messages() + app.storage.archive.reset_shown_unread_messages() # override logging settings from config (don't take care of '-q' option) if app.settings.get('verbose'): logging_helpers.set_verbose() ===================================== gajim/roster_window.py ===================================== @@ -1756,7 +1756,7 @@ class RosterWindow: Read from db the unread messages, and fire them up, and if we find very old unread messages, delete them from unread table """ - results = app.logger.get_unread_msgs() + results = app.storage.archive.get_unread_msgs() for result, shown in results: jid = result.jid additional_data = result.additional_data @@ -1770,14 +1770,14 @@ class RosterWindow: tim = float(result.time) session.roster_message(jid, result.message, tim, msg_type='chat', msg_log_id=result.log_line_id, additional_data=additional_data) - app.logger.set_shown_unread_msgs(result.log_line_id) + app.storage.archive.set_shown_unread_msgs(result.log_line_id) elif (time.time() - result.time) > 2592000: # ok, here we see that we have a message in unread messages # table that is older than a month. It is probably from someone # not in our roster for accounts we usually launch, so we will # delete this id from unread message tables. - app.logger.set_read_messages([result.log_line_id]) + app.storage.archive.set_read_messages([result.log_line_id]) def fill_contacts_and_groups_dicts(self, array, account): """ @@ -1855,7 +1855,7 @@ class RosterWindow: msg_log_ids.append(ev.msg_log_id) if msg_log_ids: - app.logger.set_read_messages(msg_log_ids) + app.storage.archive.set_read_messages(msg_log_ids) contact_list = ((event.jid.split('/')[0], event.account) for event in \ event_list) ===================================== gajim/session.py ===================================== @@ -110,7 +110,7 @@ class ChatControlSession: if not obj.properties.is_muc_pm: jid = obj.jid - obj.msg_log_id = app.logger.insert_into_logs( + obj.msg_log_id = app.storage.archive.insert_into_logs( self.conn.name, jid, obj.properties.timestamp, @@ -212,7 +212,7 @@ class ChatControlSession: read_ids = [] for msg in unread_events: read_ids.append(msg.msg_log_id) - app.logger.set_read_messages(read_ids) + app.storage.archive.set_read_messages(read_ids) app.events.remove_events(self.conn.name, fjid, types=['chat']) do_event = False else: @@ -314,7 +314,7 @@ class ChatControlSession: additional_data=additional_data) if msg_log_id: - app.logger.set_read_messages([msg_log_id]) + app.storage.archive.set_read_messages([msg_log_id]) return ===================================== pylintrc ===================================== @@ -146,7 +146,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local,gi.repository.GLib. # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. -ignored-modules= +ignored-modules=gi.repository # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. ===================================== test/broken/unit/test_sessions.py ===================================== @@ -92,7 +92,7 @@ class TestChatControlSession(unittest.TestCase): sess = self.conn.sessions[jid]['123'] # message was logged - calls = app.logger.mockGetNamedCalls('insert_into_logs') + calls = app.storage.archive.mockGetNamedCalls('insert_into_logs') self.assertEqual(1, len(calls)) # no ChatControl was open and autopopup was off @@ -118,7 +118,7 @@ class TestChatControlSession(unittest.TestCase): self.receive_chat_msg(fjid, msgtxt) # message was logged - calls = app.logger.mockGetNamedCalls('insert_into_logs') + calls = app.storage.archive.mockGetNamedCalls('insert_into_logs') self.assertEqual(2, len(calls)) # the message does not go into the event queue View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/1eee7285475b1166fc6315ef3d11477… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/1eee7285475b1166fc6315ef3d11477… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] Refactor cache storage
by Philipp Hörist 17 Sep '20

17 Sep '20

Philipp Hörist pushed to branch master at gajim / gajim Commits: 1eee7285 by lovetox at 2020-09-17T22:19:19+02:00 Refactor cache storage - Add Storage base class - Move Cache code into own module - Simplify roster cache - - - - - 23 changed files: - gajim/application.py - gajim/common/app.py - gajim/common/contacts.py - gajim/common/helpers.py - gajim/common/logger.py - gajim/common/modules/caps.py - gajim/common/modules/discovery.py - gajim/common/modules/mam.py - gajim/common/modules/message.py - gajim/common/modules/muc.py - gajim/common/modules/roster.py - gajim/common/modules/user_avatar.py - gajim/common/modules/vcard_avatars.py - gajim/common/modules/vcard_temp.py - + gajim/common/storage/__init__.py - + gajim/common/storage/base.py - + gajim/common/storage/cache.py - gajim/groupchat_control.py - gajim/gtk/avatar.py - gajim/gtk/groupchat_config.py - gajim/gtk/tooltips.py - gajim/gui_interface.py - gajim/roster_window.py Changes: ===================================== gajim/application.py ===================================== @@ -56,6 +56,7 @@ from gajim.common import logger from gajim.common.i18n import _ from gajim.common.contacts import LegacyContactsAPI from gajim.common.task_manager import TaskManager +from gajim.common.storage.cache import CacheStorage class GajimApplication(Gtk.Application): @@ -182,6 +183,8 @@ class GajimApplication(Gtk.Application): app.print_version() app.detect_dependencies() configpaths.create_paths() + app.storage.cache = CacheStorage() + app.storage.cache.init() try: app.logger = logger.Logger() app.contacts = LegacyContactsAPI() ===================================== gajim/common/app.py ===================================== @@ -71,6 +71,12 @@ plugin_manager = None # Plugins Manager logger = cast(LoggerT, None) +class Storage: + def __init__(self): + self.cache = None + +storage = Storage() + css_config = None transport_type = {} # type: Dict[str, str] ===================================== gajim/common/contacts.py ===================================== @@ -143,7 +143,7 @@ class CommonContact(XMPPEntity): # return caps for a contact that has no resources left. return False - disco_info = app.logger.get_last_disco_info(self.get_full_jid()) + disco_info = app.storage.cache.get_last_disco_info(self.get_full_jid()) if disco_info is None: return False @@ -151,7 +151,7 @@ class CommonContact(XMPPEntity): @property def uses_phone(self): - disco_info = app.logger.get_last_disco_info(self.get_full_jid()) + disco_info = app.storage.cache.get_last_disco_info(self.get_full_jid()) if disco_info is None: return False @@ -271,7 +271,7 @@ class Contact(CommonContact): if not self.is_groupchat: raise ValueError - disco_info = app.logger.get_last_disco_info(self.jid) + disco_info = app.storage.cache.get_last_disco_info(self.jid) context = 'public' if disco_info is not None and disco_info.muc_is_members_only: ===================================== gajim/common/helpers.py ===================================== @@ -898,7 +898,7 @@ def call_counter(func): return helper def get_sync_threshold(jid, archive_info): - disco_info = app.logger.get_last_disco_info(jid) + disco_info = app.storage.cache.get_last_disco_info(jid) if archive_info is None or archive_info.sync_threshold is None: if disco_info is not None and disco_info.muc_is_members_only: threshold = app.settings.get('private_room_sync_threshold') @@ -1231,7 +1231,7 @@ def get_groupchat_name(con, jid): if name: return name - disco_info = app.logger.get_last_disco_info(jid) + disco_info = app.storage.cache.get_last_disco_info(jid) if disco_info is not None: if disco_info.muc_name: return disco_info.muc_name ===================================== gajim/common/logger.py ===================================== @@ -28,7 +28,6 @@ This module allows to access the on-disk database of logs import os import sys import time -import math import datetime import calendar import json @@ -38,20 +37,16 @@ from collections import namedtuple from gi.repository import GLib -from nbxmpp.protocol import Iq -from nbxmpp.protocol import JID -from nbxmpp.structs import DiscoInfo -from nbxmpp.structs import CommonError -from nbxmpp.modules.discovery import parse_disco_info - from gajim.common import exceptions from gajim.common import app from gajim.common import configpaths from gajim.common.helpers import AdditionalDataDict from gajim.common.i18n import _ -from gajim.common.const import ( - JIDConstant, KindConstant, ShowConstant, TypeConstant, - SubscriptionConstant) +from gajim.common.const import ShowConstant +from gajim.common.const import KindConstant +from gajim.common.const import JIDConstant + +from gajim.common.storage.base import timeit LOGS_SQL_STATEMENT = ''' @@ -97,113 +92,23 @@ LOGS_SQL_STATEMENT = ''' PRAGMA user_version=5; ''' -CACHE_SQL_STATEMENT = ''' - CREATE TABLE transports_cache ( - transport TEXT UNIQUE, - type INTEGER - ); - CREATE TABLE caps_cache ( - hash_method TEXT, - hash TEXT, - data TEXT, - last_seen INTEGER - ); - CREATE TABLE last_seen_disco_info( - jid TEXT PRIMARY KEY UNIQUE, - disco_info TEXT, - last_seen INTEGER - ); - CREATE TABLE roster_entry( - account_jid_id INTEGER, - jid_id INTEGER, - name TEXT, - subscription INTEGER, - ask BOOLEAN, - avatar_sha TEXT, - PRIMARY KEY (account_jid_id, jid_id) - ); - CREATE TABLE roster_group( - account_jid_id INTEGER, - jid_id INTEGER, - group_name TEXT, - PRIMARY KEY (account_jid_id, jid_id, group_name) - ); - CREATE TABLE muc_avatars( - jid TEXT PRIMARY KEY UNIQUE, - avatar_sha TEXT - ); - PRAGMA user_version=5; - ''' log = logging.getLogger('gajim.c.logger') -def timeit(func): - def func_wrapper(self, *args, **kwargs): - start = time.time() - result = func(self, *args, **kwargs) - exec_time = (time.time() - start) * 1e3 - level = 30 if exec_time > 50 else 10 - log.log(level, 'Execution time for %s: %s ms', - func.__name__, math.ceil(exec_time)) - return result - return func_wrapper - -def _convert_disco_info(disco_info): - return parse_disco_info(Iq(node=disco_info)) - -def _adapt_disco_info(disco_info): - return str(disco_info.stanza) - -def _convert_common_error(common_error): - return CommonError.from_string(common_error) - -def _adapt_common_error(common_error): - return common_error.serialize() - -def _convert_marker(marker): - return 'received' if marker == 0 else 'displayed' - -def _jid_adapter(jid): - return str(jid) - -def _jid_converter(jid): - return JID.from_string(jid.decode()) - - -sqlite.register_converter('jid', _jid_converter) -sqlite.register_adapter(JID, _jid_adapter) - -sqlite.register_converter('disco_info', _convert_disco_info) -sqlite.register_adapter(DiscoInfo, _adapt_disco_info) - -sqlite.register_converter('common_error', _convert_common_error) -sqlite.register_adapter(CommonError, _adapt_common_error) - -sqlite.register_converter('marker', _convert_marker) - - class Logger: def __init__(self): + self._log = log self._jid_ids = {} self._jid_ids_reversed = {} self._con = None self._commit_timout_id = None self._log_db_path = configpaths.get('LOG_DB') - self._cache_db_path = configpaths.get('CACHE_DB') - - self._entity_caps_cache = {} - self._disco_info_cache = {} - self._muc_avatar_sha_cache = {} self._create_databases() self._migrate_databases() self._connect_databases() self._get_jid_ids_from_db() - self._fill_disco_info_cache() - self._fill_muc_avatar_sha_cache() - self._clean_caps_table() - self._load_caps_data() def _create_databases(self): if os.path.isdir(self._log_db_path): @@ -211,19 +116,9 @@ class Logger: self._log_db_path) sys.exit() - if os.path.isdir(self._cache_db_path): - log.error(_('%s is a directory but should be a file'), - self._cache_db_path) - sys.exit() - if not os.path.exists(self._log_db_path): - if os.path.exists(self._cache_db_path): - os.remove(self._cache_db_path) self._create(LOGS_SQL_STATEMENT, self._log_db_path) - if not os.path.exists(self._cache_db_path): - self._create(CACHE_SQL_STATEMENT, self._cache_db_path) - @staticmethod def _connect(*args, **kwargs): con = sqlite.connect(*args, **kwargs) @@ -257,10 +152,6 @@ class Logger: con = self._connect(self._log_db_path) self._migrate_logs(con) con.close() - - con = self._connect(self._cache_db_path) - self._migrate_cache(con) - con.close() except Exception: log.exception('Error') sys.exit() @@ -318,50 +209,6 @@ class Logger: ] self._execute_multiple(con, statements) - @timeit - def _migrate_cache(self, con): - if self._get_user_version(con) == 0: - # All migrations from 0.16.9 until 1.0.0 - statements = [ - 'ALTER TABLE roster_entry ADD COLUMN "avatar_sha" TEXT', - 'PRAGMA user_version=1' - ] - self._execute_multiple(con, statements) - - if self._get_user_version(con) < 2: - statements = [ - 'DROP TABLE IF EXISTS caps_cache', - 'CREATE TABLE caps_cache (hash_method TEXT, hash TEXT, data TEXT, last_seen INTEGER)', - 'PRAGMA user_version=2' - ] - self._execute_multiple(con, statements) - - if self._get_user_version(con) < 3: - statements = [ - '''CREATE TABLE last_seen_disco_info( - jid TEXT PRIMARY KEY UNIQUE, - disco_info TEXT, - last_seen INTEGER)''', - 'PRAGMA user_version=3' - ] - self._execute_multiple(con, statements) - - if self._get_user_version(con) < 4: - statements = [ - '''CREATE TABLE muc_avatars( - jid TEXT PRIMARY KEY UNIQUE, - avatar_sha TEXT)''', - 'PRAGMA user_version=4' - ] - self._execute_multiple(con, statements) - - if self._get_user_version(con) < 5: - statements = [ - 'DELETE FROM caps_cache', - 'PRAGMA user_version=5' - ] - self._execute_multiple(con, statements) - @staticmethod def _execute_multiple(con, statements): """ @@ -416,13 +263,6 @@ class Logger: self._con.create_function("get_timeout", 0, self._get_timeout) self._set_synchronous(False) - try: - self._con.execute("ATTACH DATABASE '%s' AS cache" % - self._cache_db_path.replace("'", "''")) - except Exception: - log.exception('Error') - self._con.close() - sys.exit() @timeit def _set_synchronous(self, sync): @@ -594,99 +434,6 @@ class Logger: # it's a RFC violation .... but we should not crash return None - def convert_human_transport_type_to_db_api_values(self, type_): - """ - Convert from string style to constant ints for db - """ - if type_ == 'aim': - return TypeConstant.AIM - if type_ == 'gadu-gadu': - return TypeConstant.GG - if type_ == 'http-ws': - return TypeConstant.HTTP_WS - if type_ == 'icq': - return TypeConstant.ICQ - if type_ == 'msn': - return TypeConstant.MSN - if type_ == 'qq': - return TypeConstant.QQ - if type_ == 'sms': - return TypeConstant.SMS - if type_ == 'smtp': - return TypeConstant.SMTP - if type_ in ('tlen', 'x-tlen'): - return TypeConstant.TLEN - if type_ == 'newmail': - return TypeConstant.NEWMAIL - if type_ == 'rss': - return TypeConstant.RSS - if type_ == 'weather': - return TypeConstant.WEATHER - if type_ == 'mrim': - return TypeConstant.MRIM - if type_ == 'jabber': - return TypeConstant.NO_TRANSPORT - return None - - def convert_api_values_to_human_transport_type(self, type_id): - """ - Convert from constant ints for db to string style - """ - if type_id == TypeConstant.AIM: - return 'aim' - if type_id == TypeConstant.GG: - return 'gadu-gadu' - if type_id == TypeConstant.HTTP_WS: - return 'http-ws' - if type_id == TypeConstant.ICQ: - return 'icq' - if type_id == TypeConstant.MSN: - return 'msn' - if type_id == TypeConstant.QQ: - return 'qq' - if type_id == TypeConstant.SMS: - return 'sms' - if type_id == TypeConstant.SMTP: - return 'smtp' - if type_id == TypeConstant.TLEN: - return 'tlen' - if type_id == TypeConstant.NEWMAIL: - return 'newmail' - if type_id == TypeConstant.RSS: - return 'rss' - if type_id == TypeConstant.WEATHER: - return 'weather' - if type_id == TypeConstant.MRIM: - return 'mrim' - if type_id == TypeConstant.NO_TRANSPORT: - return 'jabber' - - def convert_xmpp_sub(self, sub): - """ - Convert from string style to constant ints for db - """ - if sub == 'none': - return SubscriptionConstant.NONE - if sub == 'to': - return SubscriptionConstant.TO - if sub == 'from': - return SubscriptionConstant.FROM - if sub == 'both': - return SubscriptionConstant.BOTH - - def convert_db_sub(self, sub): - """ - Convert from constant ints for db to string style - """ - if sub == SubscriptionConstant.NONE: - return 'none' - if sub == SubscriptionConstant.TO: - return 'to' - if sub == SubscriptionConstant.FROM: - return 'from' - if sub == SubscriptionConstant.BOTH: - return 'both' - @timeit def insert_unread_events(self, message_id, jid_id): """ @@ -1043,246 +790,6 @@ class Logger: return self._con.execute( sql, tuple(jids) + (start, end)).fetchone() - @timeit - def save_transport_type(self, jid, type_): - """ - Save the type of the transport in DB - """ - type_id = self.convert_human_transport_type_to_db_api_values(type_) - if not type_id: - # unknown type - return - result = self._con.execute( - 'SELECT type from transports_cache WHERE transport = "%s"' % jid).fetchone() - if result: - if result.type == type_id: - return - sql = 'UPDATE transports_cache SET type = %d WHERE transport = "%s"' %\ - (type_id, jid) - self.simple_commit(sql) - return - sql = 'INSERT INTO transports_cache (transport, type) VALUES (?, ?)' - self._con.execute(sql, (jid, type_id)) - self._timeout_commit() - - @timeit - def get_transports_type(self): - """ - Return all the type of the transports in DB - """ - results = self._con.execute('SELECT * from transports_cache').fetchall() - if not results: - return {} - answer = {} - for result in results: - answer[result.transport] = self.convert_api_values_to_human_transport_type( - result.type) - return answer - - @timeit - def _load_caps_data(self): - ''' - Load caps cache data - ''' - rows = self._con.execute( - 'SELECT hash_method, hash, data as "data [disco_info]" ' - 'FROM caps_cache') - - for row in rows: - self._entity_caps_cache[(row.hash_method, row.hash)] = row.data - - @timeit - def add_caps_entry(self, jid, hash_method, hash_, caps_data): - self._entity_caps_cache[(hash_method, hash_)] = caps_data - - self._disco_info_cache[jid] = caps_data - - self._con.execute(''' - INSERT INTO caps_cache (hash_method, hash, data, last_seen) - VALUES (?, ?, ?, ?) - ''', (hash_method, hash_, caps_data, int(time.time()))) - self._timeout_commit() - - def get_caps_entry(self, hash_method, hash_): - return self._entity_caps_cache.get((hash_method, hash_)) - - @timeit - def update_caps_time(self, method, hash_): - sql = '''UPDATE caps_cache SET last_seen = ? - WHERE hash_method = ? and hash = ?''' - self._con.execute(sql, (int(time.time()), method, hash_)) - self._timeout_commit() - - @timeit - def _clean_caps_table(self): - """ - Remove caps which was not seen for 3 months - """ - timestamp = int(time.time()) - 3 * 30 * 24 * 3600 - self._con.execute('DELETE FROM caps_cache WHERE last_seen < ?', - (timestamp,)) - self._timeout_commit() - - @timeit - def replace_roster(self, account_name, roster_version, roster): - """ - Replace current roster in DB by a new one - - accout_name is the name of the account to change. - roster_version is the version of the new roster. - roster is the new version. - """ - # First we must reset roster_version value to ensure that the server - # sends back all the roster at the next connection if the replacement - # didn't work properly. - app.settings.set_account_setting(account_name, 'roster_version', '') - - account_jid = app.get_jid_from_account(account_name) - # Execute get_jid_id() because this ensures on new accounts that the - # jid_id will be created - self.get_jid_id(account_jid, type_=JIDConstant.NORMAL_TYPE) - - # Delete old roster - self.remove_roster(account_jid) - - # Fill roster tables with the new roster - for jid in roster: - self.add_or_update_contact(account_jid, jid, roster[jid]['name'], - roster[jid]['subscription'], roster[jid]['ask'], - roster[jid]['groups'], commit=False) - self._timeout_commit() - - # At this point, we are sure the replacement works properly so we can - # set the new roster_version value. - app.settings.set_account_setting(account_name, - 'roster_version', - roster_version) - - @timeit - def del_contact(self, account_jid, jid): - """ - Remove jid from account_jid roster - """ - try: - account_jid_id = self.get_jid_id(account_jid) - jid_id = self.get_jid_id(jid) - except exceptions.PysqliteOperationalError as e: - raise exceptions.PysqliteOperationalError(str(e)) - self._con.execute( - 'DELETE FROM roster_group WHERE account_jid_id=? AND jid_id=?', - (account_jid_id, jid_id)) - self._con.execute( - 'DELETE FROM roster_entry WHERE account_jid_id=? AND jid_id=?', - (account_jid_id, jid_id)) - self._timeout_commit() - - @timeit - def add_or_update_contact(self, account_jid, jid, name, sub, ask, groups, - commit=True): - """ - Add or update a contact from account_jid roster - """ - if sub == 'remove': - self.del_contact(account_jid, jid) - return - - try: - account_jid_id = self.get_jid_id(account_jid) - jid_id = self.get_jid_id(jid, type_=JIDConstant.NORMAL_TYPE) - except exceptions.PysqliteOperationalError as error: - raise exceptions.PysqliteOperationalError(str(error)) - - # Update groups information - # First we delete all previous groups information - sql = 'DELETE FROM roster_group WHERE account_jid_id=? AND jid_id=?' - self._con.execute(sql, (account_jid_id, jid_id)) - # Then we add all new groups information - sql = '''INSERT INTO roster_group (account_jid_id, jid_id, group_name) - VALUES (?, ?, ?)''' - for group in groups: - self._con.execute(sql, (account_jid_id, jid_id, group)) - - if name is None: - name = '' - - sql = '''REPLACE INTO roster_entry - (account_jid_id, jid_id, name, subscription, ask) - VALUES(?, ?, ?, ?, ?)''' - self._con.execute(sql, (account_jid_id, - jid_id, - name, - self.convert_xmpp_sub(sub), - bool(ask))) - if commit: - self._timeout_commit() - - @timeit - def get_roster(self, account_jid): - """ - Return the accound_jid roster in NonBlockingRoster format - """ - data = {} - account_jid_id = self.get_jid_id(account_jid, type_=JIDConstant.NORMAL_TYPE) - - # First we fill data with roster_entry information - rows = self._con.execute(''' - SELECT j.jid, re.jid_id, re.name, re.subscription, re.ask, re.avatar_sha - FROM roster_entry re, jids j - WHERE re.account_jid_id=? AND j.jid_id=re.jid_id''', (account_jid_id,)) - for row in rows: - #jid, jid_id, name, subscription, ask - jid = row.jid - name = row.name - data[jid] = {} - data[jid]['avatar_sha'] = row.avatar_sha - if name: - data[jid]['name'] = name - else: - data[jid]['name'] = None - data[jid]['subscription'] = self.convert_db_sub(row.subscription) - data[jid]['groups'] = [] - data[jid]['resources'] = {} - if row.ask: - data[jid]['ask'] = 'subscribe' - else: - data[jid]['ask'] = None - data[jid]['id'] = row.jid_id - - # Then we add group for roster entries - for jid in data: - rows = self._con.execute(''' - SELECT group_name FROM roster_group - WHERE account_jid_id=? AND jid_id=?''', - (account_jid_id, data[jid]['id'])) - for row in rows: - group_name = row.group_name - data[jid]['groups'].append(group_name) - del data[jid]['id'] - - return data - - @timeit - def remove_roster(self, account_jid): - """ - Remove the roster of an account - - :param account_jid: The jid of the account - """ - try: - jid_id = self.get_jid_id(account_jid) - except ValueError: - # This happens if the JID never made it to the Database - # because the account was never connected - return - - sql = ''' - DELETE FROM roster_entry WHERE account_jid_id = {jid_id}; - DELETE FROM roster_group WHERE account_jid_id = {jid_id}; - '''.format(jid_id=jid_id) - - self._con.executescript(sql) - self._timeout_commit() - @timeit def deduplicate_muc_message(self, account, jid, resource, timestamp, message_id): @@ -1519,71 +1026,6 @@ class Logger: self._con.execute(sql, (state, account_id, jid_id, message_id)) self._timeout_commit() - @timeit - def _fill_muc_avatar_sha_cache(self): - sql = '''SELECT jid, avatar_sha FROM muc_avatars''' - rows = self._con.execute(sql).fetchall() - for row in rows: - self._muc_avatar_sha_cache[row.jid] = row.avatar_sha - log.info('%d Avatar SHA entries loaded', len(rows)) - - @timeit - def set_avatar_sha(self, account_jid, jid, sha=None): - """ - Set the avatar sha of a jid on an account - - :param account_jid: The jid of the account - - :param jid: The jid that belongs to the avatar - - :param sha: The sha of the avatar - - """ - - account_jid_id = self.get_jid_id(account_jid) - jid_id = self.get_jid_id(jid, type_=JIDConstant.NORMAL_TYPE) - - sql = ''' - UPDATE roster_entry SET avatar_sha = ? - WHERE account_jid_id = ? AND jid_id = ? - ''' - self._con.execute(sql, (sha, account_jid_id, jid_id)) - self._timeout_commit() - - @timeit - def set_muc_avatar_sha(self, jid, sha=None): - """ - Set the avatar sha of a MUC - - :param jid: The MUC jid that belongs to the avatar - - :param sha: The sha of the avatar - - """ - - sql = '''INSERT INTO muc_avatars (jid, avatar_sha) - VALUES (?, ?)''' - - try: - self._con.execute(sql, (jid, sha)) - except sqlite.IntegrityError: - sql = 'UPDATE muc_avatars SET avatar_sha = ? WHERE jid = ?' - self._con.execute(sql, (sha, jid)) - - self._muc_avatar_sha_cache[jid] = sha - - self._timeout_commit() - - def get_muc_avatar_sha(self, jid): - """ - Get the avatar sha of a MUC - - :param jid: The MUC jid that belongs to the avatar - - """ - - return self._muc_avatar_sha_cache.get(jid) - @timeit def get_archive_infos(self, jid): """ @@ -1657,66 +1099,3 @@ class Logger: self._con.execute(sql, (jid_id,)) log.info('Reset message archive info: %s', jid) self._timeout_commit() - - @timeit - def _fill_disco_info_cache(self): - sql = '''SELECT disco_info as "disco_info [disco_info]", - jid, last_seen FROM - last_seen_disco_info''' - rows = self._con.execute(sql).fetchall() - for row in rows: - disco_info = row.disco_info._replace(timestamp=row.last_seen) - self._disco_info_cache[row.jid] = disco_info - log.info('%d DiscoInfo entries loaded', len(rows)) - - def get_last_disco_info(self, jid, max_age=0): - """ - Get last disco info from jid - - :param jid: The jid - - :param max_age: max age in seconds of the DiscoInfo record - - """ - - disco_info = self._disco_info_cache.get(jid) - if disco_info is not None: - max_timestamp = time.time() - max_age if max_age else 0 - if max_timestamp > disco_info.timestamp: - return None - return disco_info - - @timeit - def set_last_disco_info(self, jid, disco_info, cache_only=False): - """ - Get last disco info from jid - - :param jid: The jid - - :param disco_info: A DiscoInfo object - - """ - - log.info('Save disco info from %s', jid) - - if cache_only: - self._disco_info_cache[jid] = disco_info - return - - disco_exists = self.get_last_disco_info(jid) is not None - if disco_exists: - sql = '''UPDATE last_seen_disco_info SET - disco_info = ?, last_seen = ? - WHERE jid = ?''' - - self._con.execute(sql, (disco_info, disco_info.timestamp, str(jid))) - - else: - sql = '''INSERT INTO last_seen_disco_info - (jid, disco_info, last_seen) - VALUES (?, ?, ?)''' - - self._con.execute(sql, (str(jid), disco_info, disco_info.timestamp)) - - self._disco_info_cache[jid] = disco_info - self._timeout_commit() ===================================== gajim/common/modules/caps.py ===================================== @@ -101,14 +101,14 @@ class Caps(BaseModule): self._log.info('Received %s', task.entity) - disco_info = app.logger.get_caps_entry(task.entity.method, - task.entity.hash) + disco_info = app.storage.cache.get_caps_entry(task.entity.method, + task.entity.hash) if disco_info is None: self._queue_task(task) return jid = str(properties.jid) - app.logger.set_last_disco_info(jid, disco_info, cache_only=True) + app.storage.cache.set_last_disco_info(jid, disco_info, cache_only=True) app.nec.push_incoming_event( NetworkEvent('caps-update', account=self._account, @@ -143,7 +143,7 @@ class Caps(BaseModule): disco_info.jid, error) return - app.logger.add_caps_entry( + app.storage.cache.add_caps_entry( str(disco_info.jid), task.entity.method, disco_info.get_caps_hash(), ===================================== gajim/common/modules/discovery.py ===================================== @@ -153,7 +153,6 @@ class Discovery(BaseModule): jid = str(info.jid) if jid not in app.transport_type: app.transport_type[jid] = identity.type - app.logger.save_transport_type(jid, identity.type) if identity.type in self._con.available_transports: self._con.available_transports[identity.type].append(jid) @@ -200,7 +199,7 @@ class Discovery(BaseModule): def _muc_info_received(self, result, callback=None): self._log.info('MUC info received: %s', result.jid) if not is_error_result(result): - app.logger.set_last_disco_info(result.jid, result) + app.storage.cache.set_last_disco_info(result.jid, result) self._con.get_module('VCardAvatars').muc_disco_info_update(result) app.nec.push_incoming_event(NetworkEvent( 'muc-disco-update', ===================================== gajim/common/modules/mam.py ===================================== @@ -122,7 +122,7 @@ class MAM(BaseModule): archive_jid = properties.jid.bare timestamp = properties.timestamp - disco_info = app.logger.get_last_disco_info(archive_jid) + disco_info = app.storage.cache.get_last_disco_info(archive_jid) if disco_info is None: # This is the case on MUC creation # After MUC configuration we receive a configuration change ===================================== gajim/common/modules/message.py ===================================== @@ -253,7 +253,7 @@ class Message(BaseModule): message_id=event.properties.id) def _check_for_mam_compliance(self, room_jid, stanza_id): - disco_info = app.logger.get_last_disco_info(room_jid) + disco_info = app.storage.cache.get_last_disco_info(room_jid) if stanza_id is None and disco_info.mam_namespace == Namespace.MAM_2: self._log.warning('%s announces mam:2 without stanza-id', room_jid) @@ -266,7 +266,7 @@ class Message(BaseModule): return None, None if properties.type.is_groupchat: - disco_info = app.logger.get_last_disco_info( + disco_info = app.storage.cache.get_last_disco_info( properties.jid.bare) if disco_info.mam_namespace != Namespace.MAM_2: ===================================== gajim/common/modules/muc.py ===================================== @@ -141,7 +141,8 @@ class MUC(BaseModule): self._manager.add(muc_data) - disco_info = app.logger.get_last_disco_info(muc_data.jid, max_age=60) + disco_info = app.storage.cache.get_last_disco_info(muc_data.jid, + max_age=60) if disco_info is None: self._con.get_module('Discovery').disco_muc( muc_data.jid, ===================================== gajim/common/modules/roster.py ===================================== @@ -45,8 +45,7 @@ class Roster(BaseModule): def load_roster(self): self._log.info('Load from database') - account_jid = self._con.get_own_jid().bare - data = app.logger.get_roster(account_jid) + data = app.storage.cache.load_roster(self._account) if data: self.set_raw(data) for jid, item in self._data.items(): @@ -71,6 +70,9 @@ class Roster(BaseModule): roster=self._data.copy(), received_from_server=False)) + def _store_roster(self): + app.storage.cache.store_roster(self._account, self._data) + def request_roster(self): version = None features = self._con.connection.features @@ -99,7 +101,10 @@ class Roster(BaseModule): version = self._parse_roster(stanza) self._log.info('New version: %s', version) - app.logger.replace_roster(self._account, version, self._data) + self._store_roster() + app.settings.set_account_setting(self._account, + 'roster_version', + version) received_from_server = True @@ -135,10 +140,8 @@ class Roster(BaseModule): ask=attrs['ask'], groups=attrs['groups'], avatar_sha=None)) - account_jid = self._con.get_own_jid().bare - app.logger.add_or_update_contact( - account_jid, item.jid, attrs['name'], - attrs['subscription'], attrs['ask'], attrs['groups']) + + self._store_roster() self._log.info('New version: %s', version) app.settings.set_account_setting(self._account, @@ -238,13 +241,8 @@ class Roster(BaseModule): 'subscription': 'none', 'groups': ['Not in contact list']} - account_jid = self._con.get_own_jid().bare - app.logger.add_or_update_contact( - account_jid, jid, - self._data[jid]['name'], - self._data[jid]['subscription'], - self._data[jid]['ask'], - self._data[jid]['groups']) + + self._store_roster() def _get_item_data(self, jid, dataname): """ @@ -369,6 +367,13 @@ class Roster(BaseModule): 'avatar_sha': None } + def set_avatar_sha(self, jid, sha): + if jid not in self._data: + return + + self._data[jid]['avatar_sha'] = sha + self._store_roster() + def get_instance(*args, **kwargs): return Roster(*args, **kwargs), 'Roster' ===================================== gajim/common/modules/user_avatar.py ===================================== @@ -40,13 +40,12 @@ class UserAvatar(BaseModule): data = properties.pubsub_event.data jid = str(properties.jid) - own_jid = self._con.get_own_jid().bare if data is None: # Remove avatar self._log.info('Remove: %s', jid) app.contacts.set_avatar(self._account, jid, None) - app.logger.set_avatar_sha(own_jid, jid, None) + self._con.get_module('Roster').set_avatar_sha(jid, None) app.interface.update_avatar(self._account, jid) else: if properties.is_self_message: @@ -76,8 +75,8 @@ class UserAvatar(BaseModule): 'avatar_sha', result.sha) else: - own_jid = self._con.get_own_jid().bare - app.logger.set_avatar_sha(own_jid, str(result.jid), result.sha) + self._con.get_module('Roster').set_avatar_sha(str(result.jid), + result.sha) app.contacts.set_avatar(self._account, str(result.jid), result.sha) app.interface.update_avatar(self._account, str(result.jid)) ===================================== gajim/common/modules/vcard_avatars.py ===================================== @@ -132,7 +132,6 @@ class VCardAvatars(BaseModule): room) def _process_update(self, jid, state, avatar_sha, room): - acc_jid = self._con.get_own_jid().bare if state == AvatarState.EMPTY: # Empty <photo/> tag, means no avatar is advertised self._log.info('%s has no avatar published', jid) @@ -142,9 +141,9 @@ class VCardAvatars(BaseModule): app.contacts.set_avatar(self._account, jid, None) if room: - app.logger.set_muc_avatar_sha(jid, None) + app.storage.cache.set_muc_avatar_sha(jid, None) else: - app.logger.set_avatar_sha(acc_jid, jid, None) + self._con.get_module('Roster').set_avatar_sha(jid, None) app.interface.update_avatar(self._account, jid, room_avatar=room) else: self._log.info('Update: %s %s', jid, avatar_sha) @@ -158,9 +157,10 @@ class VCardAvatars(BaseModule): # Check if the avatar is already in storage self._log.info('Found avatar in storage') if room: - app.logger.set_muc_avatar_sha(jid, avatar_sha) + app.storage.cache.set_muc_avatar_sha(jid, avatar_sha) else: - app.logger.set_avatar_sha(acc_jid, jid, avatar_sha) + self._con.get_module('Roster').set_avatar_sha(jid, + avatar_sha) app.contacts.set_avatar(self._account, jid, avatar_sha) app.interface.update_avatar( self._account, jid, room_avatar=room) ===================================== gajim/common/modules/vcard_temp.py ===================================== @@ -266,7 +266,7 @@ class VCardTemp(BaseModule): app.interface.save_avatar(photo_decoded) self._log.info('Received: %s %s', jid, avatar_sha) - app.logger.set_muc_avatar_sha(jid, avatar_sha) + app.storage.cache.set_muc_avatar_sha(jid, avatar_sha) app.contacts.set_avatar(self._account, jid, avatar_sha) app.interface.update_avatar(self._account, jid, room_avatar=True) @@ -292,8 +292,7 @@ class VCardTemp(BaseModule): app.interface.update_avatar(contact=contact) else: self._log.info('Received: %s %s', jid, avatar_sha) - own_jid = self._con.get_own_jid().bare - app.logger.set_avatar_sha(own_jid, jid, avatar_sha) + self._con.get_module('Roster').set_avatar_sha(jid, avatar_sha) app.contacts.set_avatar(self._account, jid, avatar_sha) app.interface.update_avatar(self._account, jid) ===================================== gajim/common/storage/__init__.py ===================================== ===================================== gajim/common/storage/base.py ===================================== @@ -0,0 +1,179 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see <http://www.gnu.org/licenses/>. + +import sys +import math +import time +import sqlite3 + +from gi.repository import GLib + +from nbxmpp.protocol import Iq +from nbxmpp.protocol import JID +from nbxmpp.structs import DiscoInfo +from nbxmpp.structs import CommonError +from nbxmpp.modules.discovery import parse_disco_info + + +def timeit(func): + def func_wrapper(self, *args, **kwargs): + start = time.time() + result = func(self, *args, **kwargs) + exec_time = (time.time() - start) * 1e3 + level = 30 if exec_time > 50 else 10 + self._log.log(level, + 'Execution time for %s: %s ms', + func.__name__, + math.ceil(exec_time)) + return result + return func_wrapper + + +def _convert_common_error(common_error): + return CommonError.from_string(common_error) + +def _adapt_common_error(common_error): + return common_error.serialize() + +sqlite3.register_converter('common_error', _convert_common_error) +sqlite3.register_adapter(CommonError, _adapt_common_error) + +def _convert_marker(marker): + return 'received' if marker == 0 else 'displayed' + +sqlite3.register_converter('marker', _convert_marker) + +def _jid_adapter(jid): + return str(jid) + +def _jid_converter(jid): + return JID.from_string(jid.decode()) + +sqlite3.register_converter('jid', _jid_converter) +sqlite3.register_adapter(JID, _jid_adapter) + +def _convert_disco_info(disco_info): + return parse_disco_info(Iq(node=disco_info)) + +def _adapt_disco_info(disco_info): + return str(disco_info.stanza) + +sqlite3.register_converter('disco_info', _convert_disco_info) +sqlite3.register_adapter(DiscoInfo, _adapt_disco_info) + + +class SqliteStorage: + ''' + Base Storage Class + ''' + + def __init__(self, + log, + path, + create_statement, + commit_delay=500): + + self._log = log + self._path = path + self._create_statement = create_statement + self._commit_delay = commit_delay + self._con = None + self._commit_source_id = None + + def init(self, **kwargs): + if self._path.exists(): + if not self._path.is_file(): + sys.exit('%s must be a file', self._path) + self._con = self._connect(**kwargs) + + else: + self._con = self._create_storage(**kwargs) + + self._migrate_storage() + + def _set_journal_mode(self, mode): + self._con.execute(f'PRAGMA journal_mode={mode}') + + def _set_synchronous(self, mode): + self._con.execute(f'PRAGMA synchronous={mode}') + + @property + def user_version(self) -> int: + return self._con.execute('PRAGMA user_version').fetchone()[0] + + def _connect(self, **kwargs): + return sqlite3.connect(self._path, **kwargs) + + def _create_storage(self, **kwargs): + self._log.info('Creating %s', self._path) + con = self._connect(**kwargs) + self._path.chmod(0o600) + + try: + con.executescript(self._create_statement) + except Exception: + self._log.exception('Error') + con.close() + self._path.unlink() + sys.exit('Failed creating storage') + + con.commit() + return con + + def _reinit_storage(self): + if self._con is not None: + self._con.close() + self._path.unlink() + self.init() + + def _migrate_storage(self): + try: + self._migrate() + except Exception: + self._con.close() + self._log.exception('Error') + sys.exit() + + def _migrate(self): + raise NotImplementedError + + def _execute_multiple(self, statements): + """ + Execute multiple statements with the option to fail on duplicates + but still continue + """ + for sql in statements: + try: + self._con.execute(sql) + self._con.commit() + except sqlite3.OperationalError as error: + if str(error).startswith('duplicate column name:'): + self._log.info(error) + else: + self._con.close() + self._log.exception('Error') + sys.exit() + + @timeit + def _commit(self): + self._commit_source_id = None + self._con.commit() + return False + + def _delayed_commit(self): + if self._commit_source_id is not None: + return + + self._commit_source_id = GLib.timeout_add(self._commit_delay, + self._commit) ===================================== gajim/common/storage/cache.py ===================================== @@ -0,0 +1,265 @@ +# This file is part of Gajim. +# +# Gajim is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published +# by the Free Software Foundation; version 3 only. +# +# Gajim is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Gajim. If not, see <http://www.gnu.org/licenses/>. + +import json +import time +import sqlite3 +import logging +from pathlib import Path +from collections import namedtuple + +from gajim.common import configpaths +from gajim.common.storage.base import SqliteStorage +from gajim.common.storage.base import timeit + + +CURRENT_USER_VERSION = 6 + +CACHE_SQL_STATEMENT = ''' + CREATE TABLE caps_cache ( + hash_method TEXT, + hash TEXT, + data TEXT, + last_seen INTEGER + ); + CREATE TABLE last_seen_disco_info( + jid TEXT PRIMARY KEY UNIQUE, + disco_info TEXT, + last_seen INTEGER + ); + CREATE TABLE roster( + account TEXT PRIMARY KEY UNIQUE, + roster TEXT + ); + CREATE TABLE muc_avatars( + jid TEXT PRIMARY KEY UNIQUE, + avatar_sha TEXT + ); + PRAGMA user_version=%s; + ''' % CURRENT_USER_VERSION + +log = logging.getLogger('gajim.c.storage.cache') + + +class CacheStorage(SqliteStorage): + def __init__(self): + SqliteStorage.__init__(self, + log, + Path(configpaths.get('CACHE_DB')), + CACHE_SQL_STATEMENT) + + self._entity_caps_cache = {} + self._disco_info_cache = {} + self._muc_avatar_sha_cache = {} + + def init(self, **kwargs): + SqliteStorage.init(self, + detect_types=sqlite3.PARSE_COLNAMES) + self._set_journal_mode('WAL') + self._con.row_factory = self._namedtuple_factory + + self._fill_disco_info_cache() + self._fill_muc_avatar_sha_cache() + self._clean_caps_table() + self._load_caps_data() + + @staticmethod + def _namedtuple_factory(cursor, row): + fields = [col[0] for col in cursor.description] + Row = namedtuple("Row", fields) + return Row(*row) + + def _migrate(self): + user_version = self.user_version + if user_version > CURRENT_USER_VERSION: + # Gajim was downgraded, reinit the storage + self._reinit_storage() + return + + if user_version < 6: + self._reinit_storage() + + @timeit + def _load_caps_data(self): + rows = self._con.execute( + 'SELECT hash_method, hash, data as "data [disco_info]" ' + 'FROM caps_cache') + + for row in rows: + self._entity_caps_cache[(row.hash_method, row.hash)] = row.data + + @timeit + def add_caps_entry(self, jid, hash_method, hash_, caps_data): + self._entity_caps_cache[(hash_method, hash_)] = caps_data + + self._disco_info_cache[jid] = caps_data + + self._con.execute(''' + INSERT INTO caps_cache (hash_method, hash, data, last_seen) + VALUES (?, ?, ?, ?) + ''', (hash_method, hash_, caps_data, int(time.time()))) + self._delayed_commit() + + def get_caps_entry(self, hash_method, hash_): + return self._entity_caps_cache.get((hash_method, hash_)) + + @timeit + def update_caps_time(self, method, hash_): + sql = '''UPDATE caps_cache SET last_seen = ? + WHERE hash_method = ? and hash = ?''' + self._con.execute(sql, (int(time.time()), method, hash_)) + self._delayed_commit() + + @timeit + def _clean_caps_table(self): + """ + Remove caps which was not seen for 3 months + """ + timestamp = int(time.time()) - 3 * 30 * 24 * 3600 + self._con.execute('DELETE FROM caps_cache WHERE last_seen < ?', + (timestamp,)) + self._delayed_commit() + + @timeit + def _fill_disco_info_cache(self): + sql = '''SELECT disco_info as "disco_info [disco_info]", + jid, last_seen FROM + last_seen_disco_info''' + rows = self._con.execute(sql).fetchall() + for row in rows: + disco_info = row.disco_info._replace(timestamp=row.last_seen) + self._disco_info_cache[row.jid] = disco_info + log.info('%d DiscoInfo entries loaded', len(rows)) + + def get_last_disco_info(self, jid, max_age=0): + """ + Get last disco info from jid + + :param jid: The jid + + :param max_age: max age in seconds of the DiscoInfo record + + """ + + disco_info = self._disco_info_cache.get(jid) + if disco_info is not None: + max_timestamp = time.time() - max_age if max_age else 0 + if max_timestamp > disco_info.timestamp: + return None + return disco_info + + @timeit + def set_last_disco_info(self, jid, disco_info, cache_only=False): + """ + Get last disco info from jid + + :param jid: The jid + + :param disco_info: A DiscoInfo object + + """ + + log.info('Save disco info from %s', jid) + + if cache_only: + self._disco_info_cache[jid] = disco_info + return + + disco_exists = self.get_last_disco_info(jid) is not None + if disco_exists: + sql = '''UPDATE last_seen_disco_info SET + disco_info = ?, last_seen = ? + WHERE jid = ?''' + + self._con.execute(sql, (disco_info, disco_info.timestamp, str(jid))) + + else: + sql = '''INSERT INTO last_seen_disco_info + (jid, disco_info, last_seen) + VALUES (?, ?, ?)''' + + self._con.execute(sql, (str(jid), disco_info, disco_info.timestamp)) + + self._disco_info_cache[jid] = disco_info + self._delayed_commit() + + @timeit + def store_roster(self, account, roster): + serialized = json.dumps(roster) + + insert_sql = 'INSERT INTO roster(account, roster) VALUES(?, ?)' + update_sql = 'UPDATE roster SET roster = ? WHERE account = ?' + + try: + self._con.execute(insert_sql, (account, serialized)) + except sqlite3.IntegrityError: + self._con.execute(update_sql, (serialized, account)) + + self._delayed_commit() + + @timeit + def load_roster(self, account): + select_sql = 'SELECT roster FROM roster WHERE account = ?' + result = self._con.execute(select_sql, (account,)).fetchone() + if result is None: + return None + return json.loads(result.roster) + + @timeit + def remove_roster(self, account): + delete_sql = 'DELETE FROM roster WHERE account = ?' + self._con.execute(delete_sql, (account,)) + self._commit() + + @timeit + def _fill_muc_avatar_sha_cache(self): + sql = '''SELECT jid, avatar_sha FROM muc_avatars''' + rows = self._con.execute(sql).fetchall() + for row in rows: + self._muc_avatar_sha_cache[row.jid] = row.avatar_sha + log.info('%d Avatar SHA entries loaded', len(rows)) + + @timeit + def set_muc_avatar_sha(self, jid, sha=None): + """ + Set the avatar sha of a MUC + + :param jid: The MUC jid that belongs to the avatar + + :param sha: The sha of the avatar + + """ + + sql = '''INSERT INTO muc_avatars (jid, avatar_sha) + VALUES (?, ?)''' + + try: + self._con.execute(sql, (jid, sha)) + except sqlite3.IntegrityError: + sql = 'UPDATE muc_avatars SET avatar_sha = ? WHERE jid = ?' + self._con.execute(sql, (sha, jid)) + + self._muc_avatar_sha_cache[jid] = sha + + self._delayed_commit() + + def get_muc_avatar_sha(self, jid): + """ + Get the avatar sha of a MUC + + :param jid: The MUC jid that belongs to the avatar + + """ + + return self._muc_avatar_sha_cache.get(jid) ===================================== gajim/groupchat_control.py ===================================== @@ -266,7 +266,7 @@ class GroupchatControl(ChatControlBase): @property def disco_info(self): - return app.logger.get_last_disco_info(self.contact.jid) + return app.storage.cache.get_last_disco_info(self.contact.jid) def add_actions(self): super().add_actions() ===================================== gajim/gtk/avatar.py ===================================== @@ -239,7 +239,7 @@ class AvatarStorage(metaclass=Singleton): if surface is not None: return surface - avatar_sha = app.logger.get_muc_avatar_sha(jid) + avatar_sha = app.storage.cache.get_muc_avatar_sha(jid) if avatar_sha is not None: surface = self.surface_from_filename(avatar_sha, size, scale) if surface is None: ===================================== gajim/gtk/groupchat_config.py ===================================== @@ -52,7 +52,7 @@ class GroupchatConfig(Gtk.ApplicationWindow): self._ui.add_button.set_sensitive(True) self._ui.add_button.set_tooltip_text('') - disco_info = app.logger.get_last_disco_info(self.jid) + disco_info = app.storage.cache.get_last_disco_info(self.jid) visible = disco_info.supports(Namespace.REGISTER) self._ui.reserved_name_column.set_visible(visible) self._ui.info_button.set_sensitive(False) ===================================== gajim/gtk/tooltips.py ===================================== @@ -357,7 +357,7 @@ class RosterTooltip(StatusTable): else: # only one resource if contact.is_groupchat: - disco_info = app.logger.get_last_disco_info(contact.jid) + disco_info = app.storage.cache.get_last_disco_info(contact.jid) if disco_info is not None: description = disco_info.muc_description if description: ===================================== gajim/gui_interface.py ===================================== @@ -1321,7 +1321,7 @@ class Interface: def create_groupchat_control(self, account, room_jid, muc_data, minimize=False): - avatar_sha = app.logger.get_muc_avatar_sha(room_jid) + avatar_sha = app.storage.cache.get_muc_avatar_sha(room_jid) contact = app.contacts.create_contact(jid=room_jid, account=account, groups=[_('Group chats')], @@ -1653,7 +1653,7 @@ class Interface: if app.settings.get_account_setting(account, 'active'): self.disable_account(account) - app.logger.remove_roster(app.get_jid_from_account(account)) + app.storage.cache.remove_roster(account) # Delete password must be before del_per() because it calls set_per() # which would recreate the account with defaults values if not found passwords.delete_password(account) @@ -2103,9 +2103,6 @@ class Interface: self.make_regexps() - # get transports type from DB - app.transport_type = app.logger.get_transports_type() - self.last_ftwindow_update = 0 self._network_monitor = Gio.NetworkMonitor.get_default() ===================================== gajim/roster_window.py ===================================== @@ -1155,7 +1155,7 @@ class RosterWindow: status_span = '\n<span size="small" style="italic" ' \ 'alpha="70%">{}</span>' if contact.is_groupchat: - disco_info = app.logger.get_last_disco_info(contact.jid) + disco_info = app.storage.cache.get_last_disco_info(contact.jid) if disco_info is not None: description = disco_info.muc_description if description: View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/1eee7285475b1166fc6315ef3d114775… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/1eee7285475b1166fc6315ef3d114775… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] Settings: Allow context to be defined per setting
by Philipp Hörist 17 Sep '20

17 Sep '20

Philipp Hörist pushed to branch master at gajim / gajim Commits: 878a1e47 by lovetox at 2020-09-17T07:13:15+02:00 Settings: Allow context to be defined per setting - - - - - 2 changed files: - gajim/gtk/const.py - gajim/gtk/settings.py Changes: ===================================== gajim/gtk/const.py ===================================== @@ -22,7 +22,7 @@ from enum import unique Filter = namedtuple('Filter', 'name pattern default') Setting = namedtuple('Setting', 'kind label type value name callback data desc ' - 'bind inverted enabled_func props') + 'bind inverted enabled_func context props') Setting.__new__.__defaults__ = (None,) * len(Setting._fields) # type: ignore @unique ===================================== gajim/gtk/settings.py ===================================== @@ -73,12 +73,11 @@ class SettingsDialog(Gtk.ApplicationWindow): class SettingsBox(Gtk.ListBox): - def __init__(self, account=None, jid=None, context=None, extend=None): + def __init__(self, account=None, jid=None, extend=None): Gtk.ListBox.__init__(self) self.get_style_context().add_class('settings-box') self.account = account self.jid = jid - self.context = context self.named_settings = {} self.map = { @@ -116,13 +115,11 @@ class SettingsBox(Gtk.ListBox): if setting.props is not None: listitem = self.map[setting.kind](self.account, self.jid, - self.context, *setting[1:-1], **setting.props) else: listitem = self.map[setting.kind](self.account, self.jid, - self.context, *setting[1:-1]) if setting.name is not None: @@ -141,7 +138,6 @@ class GenericSetting(Gtk.ListBoxRow): def __init__(self, account, jid, - context, label, type_, value, @@ -151,7 +147,9 @@ class GenericSetting(Gtk.ListBoxRow): desc, bind, inverted, - enabled_func): + enabled_func, + context): + Gtk.ListBoxRow.__init__(self) self._grid = Gtk.Grid() self._grid.set_size_request(-1, 30) View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/878a1e47cc6597c501c5ca8df5497b9e… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/878a1e47cc6597c501c5ca8df5497b9e… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] Settings: Add missing return
by Philipp Hörist 17 Sep '20

17 Sep '20

Philipp Hörist pushed to branch master at gajim / gajim Commits: 5351b2bf by lovetox at 2020-09-17T06:36:31+02:00 Settings: Add missing return - - - - - 1 changed file: - gajim/gtk/settings.py Changes: ===================================== gajim/gtk/settings.py ===================================== @@ -267,7 +267,7 @@ class GenericSetting(Gtk.ListBoxRow): return app.settings.get_contact_setting(account, jid, value) if type_ == SettingType.GROUP_CHAT: - value = app.settings.get_group_chat_setting( + return app.settings.get_group_chat_setting( account, jid, value, context) if type_ == SettingType.CONFIG: View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/5351b2bf4564131c8f10ec035ee0f0cc… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/commit/5351b2bf4564131c8f10ec035ee0f0cc… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/gajim][master] 2 commits: Settings: Add contact and group chat settings
by Philipp Hörist 16 Sep '20

16 Sep '20

Philipp Hörist pushed to branch master at gajim / gajim Commits: d8e6a6e4 by lovetox at 2020-09-16T21:30:06+02:00 Settings: Add contact and group chat settings - - - - - 47a2dd56 by lovetox at 2020-09-16T22:06:08+02:00 Settings: Allow bind to all setting types - - - - - 3 changed files: - gajim/gtk/accounts.py - gajim/gtk/const.py - gajim/gtk/settings.py Changes: ===================================== gajim/gtk/accounts.py ===================================== @@ -610,7 +610,7 @@ class GeneralPage(GenericSettingPage): desc=_('Recognize your account by color')), Setting(SettingKind.LOGIN, _('Login'), SettingType.DIALOG, - bind='anonymous_auth', + bind='account::anonymous_auth', inverted=True, props={'dialog': LoginDialog}), @@ -863,7 +863,7 @@ class PriorityDialog(SettingsDialog): Setting(SettingKind.SPIN, _('Priority'), SettingType.ACCOUNT_CONFIG, 'priority', - bind='adjust_priority_with_status', + bind='account::adjust_priority_with_status', inverted=True, props={'range_': range_}), ] @@ -894,15 +894,15 @@ class CutstomHostnameDialog(SettingsDialog): Setting(SettingKind.ENTRY, _('Hostname'), SettingType.ACCOUNT_CONFIG, 'custom_host', - bind='use_custom_host'), + bind='account::use_custom_host'), Setting(SettingKind.ENTRY, _('Port'), SettingType.ACCOUNT_CONFIG, 'custom_port', - bind='use_custom_host'), + bind='account::use_custom_host'), Setting(SettingKind.COMBO, _('Type'), SettingType.ACCOUNT_CONFIG, 'custom_type', - bind='use_custom_host', + bind='account::use_custom_host', props={'combo_items': type_values}), ] @@ -932,7 +932,7 @@ class LoginDialog(SettingsDialog): settings = [ Setting(SettingKind.ENTRY, _('Password'), SettingType.ACCOUNT_CONFIG, 'password', - bind='savepass'), + bind='account::savepass'), Setting(SettingKind.SWITCH, _('Save Password'), SettingType.ACCOUNT_CONFIG, 'savepass'), ===================================== gajim/gtk/const.py ===================================== @@ -65,11 +65,13 @@ class SettingKind(IntEnum): @unique class SettingType(IntEnum): - ACCOUNT_CONFIG = 0 - CONFIG = 1 - VALUE = 2 - ACTION = 3 - DIALOG = 4 + CONFIG = 0 + ACCOUNT_CONFIG = 1 + CONTACT = 2 + GROUP_CHAT = 3 + VALUE = 4 + ACTION = 5 + DIALOG = 6 class ControlType(Enum): ===================================== gajim/gtk/settings.py ===================================== @@ -73,10 +73,12 @@ class SettingsDialog(Gtk.ApplicationWindow): class SettingsBox(Gtk.ListBox): - def __init__(self, account, extend=None): + def __init__(self, account=None, jid=None, context=None, extend=None): Gtk.ListBox.__init__(self) self.get_style_context().add_class('settings-box') self.account = account + self.jid = jid + self.context = context self.named_settings = {} self.map = { @@ -112,10 +114,16 @@ class SettingsBox(Gtk.ListBox): def add_setting(self, setting): if not isinstance(setting, Gtk.ListBoxRow): if setting.props is not None: - listitem = self.map[setting.kind]( - self.account, *setting[1:-1], **setting.props) + listitem = self.map[setting.kind](self.account, + self.jid, + self.context, + *setting[1:-1], + **setting.props) else: - listitem = self.map[setting.kind](self.account, *setting[1:-1]) + listitem = self.map[setting.kind](self.account, + self.jid, + self.context, + *setting[1:-1]) if setting.name is not None: self.named_settings[setting.name] = listitem @@ -132,6 +140,8 @@ class SettingsBox(Gtk.ListBox): class GenericSetting(Gtk.ListBoxRow): def __init__(self, account, + jid, + context, label, type_, value, @@ -153,6 +163,8 @@ class GenericSetting(Gtk.ListBoxRow): self.data = data self.label = label self.account = account + self.jid = jid + self.context = context self.name = name self.bind = bind self.inverted = inverted @@ -197,31 +209,67 @@ class GenericSetting(Gtk.ListBoxRow): if self.bind is None: return - app.settings.bind_signal(self.bind, + bind_setting_type, setting, account, jid = self._parse_bind() + + app.settings.bind_signal(setting, self, 'set_sensitive', - account=self.account, + account=account, + jid=jid, inverted=self.inverted) - if self.account is not None: - value = app.settings.get_account_setting(self.account, self.bind) + if bind_setting_type == SettingType.CONTACT: + value = app.settings.get_contact_setting(account, jid, setting) + + elif bind_setting_type == SettingType.GROUP_CHAT: + value = app.settings.get_group_chat_setting(account, jid, setting) + + elif bind_setting_type == SettingType.ACCOUNT_CONFIG: + value = app.settings.get_account_setting(account, setting) + else: - value = app.settings.get(self.bind) + value = app.settings.get(setting) if self.inverted: value = not value self.set_sensitive(value) + def _parse_bind(self): + if '::' not in self.bind: + return SettingType.CONFIG, self.bind, None, None + + bind_setting_type, setting = self.bind.split('::') + if bind_setting_type == 'account': + return SettingType.ACCOUNT_CONFIG, setting, self.account, None + + if bind_setting_type == 'contact': + return SettingType.CONTACT, setting, self.account, self.jid + + if bind_setting_type == 'group_chat': + return SettingType.GROUP_CHAT, setting, self.account, self.jid + raise ValueError(f'Invalid bind argument: {self.bind}') + def get_value(self): - return self.__get_value(self.type_, self.value, self.account) + return self.__get_value(self.type_, + self.value, + self.account, + self.jid, + self.context) @staticmethod - def __get_value(type_, value, account): + def __get_value(type_, value, account, jid, context): if value is None: return None if type_ == SettingType.VALUE: return value + if type_ == SettingType.CONTACT: + return app.settings.get_contact_setting(account, jid, value) + + if type_ == SettingType.GROUP_CHAT: + value = app.settings.get_group_chat_setting( + account, jid, value, context) + if type_ == SettingType.CONFIG: return app.settings.get(value) @@ -244,7 +292,8 @@ class GenericSetting(Gtk.ListBoxRow): def set_value(self, state): if self.type_ == SettingType.CONFIG: app.settings.set(self.value, state) - if self.type_ == SettingType.ACCOUNT_CONFIG: + + elif self.type_ == SettingType.ACCOUNT_CONFIG: if self.value == 'password': passwords.save_password(self.account, state) if self.value == 'no_log_for': @@ -254,6 +303,14 @@ class GenericSetting(Gtk.ListBoxRow): self.value, state) + elif self.type_ == SettingType.CONTACT: + app.settings.set_contact_setting( + self.account, self.jid, self.value, state) + + elif self.type_ == SettingType.GROUP_CHAT: + app.settings.set_group_chat_setting( + self.account, self.jid, self.value, state, self.context) + if self.callback is not None: self.callback(state, self.data) View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/ba321d4938db2ab1d1909360abb30c7… -- View it on GitLab: https://dev.gajim.org/gajim/gajim/-/compare/ba321d4938db2ab1d1909360abb30c7… You're receiving this email because of your account on dev.gajim.org.

1 0

[Git][gajim/python-nbxmpp][tasks] Fix pylint error
by Philipp Hörist 16 Sep '20

16 Sep '20

Philipp Hörist pushed to branch tasks at gajim / python-nbxmpp Commits: e59383ae by lovetox at 2020-09-16T14:24:55+02:00 Fix pylint error - - - - - 1 changed file: - nbxmpp/task.py Changes: ===================================== nbxmpp/task.py ===================================== @@ -207,7 +207,7 @@ class Task: def finish(self): if self._error is not None: - raise self._error + raise self._error # pylint: disable=raising-bad-type return self._result def set_user_data(self, user_data): View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/e59383ae5d02c29e8d3cff8d… -- View it on GitLab: https://dev.gajim.org/gajim/python-nbxmpp/-/commit/e59383ae5d02c29e8d3cff8d… You're receiving this email because of your account on dev.gajim.org.

1 0

Jump to page:

2026

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

Commits