Request 2793 (accepted)

Overview

Request 2793 (accepted)

Without patch

Created by vanmeeuwen about 3 years ago
In state accepted

Submit roundcubemail-plugins-kolab

Submit package Kolab:Winterfell / roundcubema...gins-kolab to package Kolab:16 / roundcubemail-plugins-kolab

roundcubemail-plugins-kolab.spec Changed

0001-Fix-kolab-cache-sync-issues.patch Deleted

@@ -1,537 +0,0 @@
-From 23c76d2ef80b35acf8d86ae4e4391020918998cf Mon Sep 17 00:00:00 2001
-From: Aleksander Machniak <machniak@kolabsys.com>
-Date: Fri, 16 Oct 2020 11:01:12 +0200
-Subject: [PATCH] Fix kolab cache sync issues
-
-Summary: Use QRESYNC, get rid of "scheduled" cache reset, other small improvements
-
-Reviewers: #roundcube_kolab_plugins_developers
-
-Subscribers: #roundcube_kolab_plugins_developers
-
-Differential Revision: https://git.kolab.org/D1726
----
- plugins/libkolab/config.inc.php.dist         |   4 -
- plugins/libkolab/lib/kolab_storage_cache.php | 404 ++++++++++++++-----
- 2 files changed, 295 insertions(+), 113 deletions(-)
-
-diff --git a/plugins/libkolab/config.inc.php.dist b/plugins/libkolab/config.inc.php.dist
-index 81f4ea80..9801af37 100644
---- a/plugins/libkolab/config.inc.php.dist
-+++ b/plugins/libkolab/config.inc.php.dist
-@@ -5,10 +5,6 @@
- // Enable caching of Kolab objects in local database
- $config['kolab_cache'] = true;
- 
--// Cache refresh interval (default is 12 hours)
--// after this period, cache is forced to synchronize with IMAP
--$config['kolab_cache_refresh'] = '12h';
--
- // Specify format version to write Kolab objects (must be a string value!)
- $config['kolab_format_version']  = '3.0';
- 
-diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
-index 8f682e54..18b5e5f2 100644
---- a/plugins/libkolab/lib/kolab_storage_cache.php
-+++ b/plugins/libkolab/lib/kolab_storage_cache.php
-@@ -27,8 +27,6 @@ class kolab_storage_cache
-     const DB_DATE_FORMAT = 'Y-m-d H:i:s';
-     const MAX_RECORDS    = 500;
- 
--    public $sync_complete = false;
--
-     protected $db;
-     protected $imap;
-     protected $folder;
-@@ -42,7 +40,6 @@ class kolab_storage_cache
-     protected $synclock = false;
-     protected $ready = false;
-     protected $cache_table;
--    protected $cache_refresh = 3600;
-     protected $folders_table;
-     protected $max_sql_packet;
-     protected $max_sync_lock_time = 600;
-@@ -85,7 +82,6 @@ class kolab_storage_cache
-         $this->imap = $rcmail->get_storage();
-         $this->enabled = $rcmail->config->get('kolab_cache', false);
-         $this->folders_table = $this->db->table_name('kolab_folders');
--        $this->cache_refresh = get_offset_sec($rcmail->config->get('kolab_cache_refresh', '12h'));
-         $this->server_timezone = new DateTimeZone(date_default_timezone_get());
- 
-         if ($this->enabled) {
-@@ -174,16 +170,6 @@ class kolab_storage_cache
-         if ($this->synched)
-             return;
- 
--        // increase time limit
--        @set_time_limit($this->max_sync_lock_time - 60);
--
--        // get effective time limit we have for synchronization (~70% of the execution time)
--        $time_limit = ini_get('max_execution_time') * 0.7;
--        $sync_start = time();
--
--        // assume sync will be completed
--        $this->sync_complete = true;
--
-         if (!$this->ready) {
-             // kolab cache is disabled, synchronize IMAP mailbox cache only
-             $this->imap_mode(true);
-@@ -191,124 +177,312 @@ class kolab_storage_cache
-             $this->imap_mode(false);
-         }
-         else {
-+            $this->sync_start = time();
-+
-             // read cached folder metadata
-             $this->_read_folder_data();
- 
-+            // Read folder data from IMAP
-+            $ctag = $this->folder->get_ctag();
-+
-+            // Validate current ctag
-+            list($uidvalidity, $highestmodseq, $uidnext) = explode('-', $ctag);
-+
-+            if (empty($uidvalidity) || empty($highestmodseq)) {
-+                rcube::raise_error(array(
-+                    'code' => 900,
-+                    'message' => "Failed to sync the kolab cache (Invalid ctag)"
-+                ), true);
-+            }
-             // check cache status ($this->metadata is set in _read_folder_data())
--            if (  empty($this->metadata['ctag']) ||
--                  empty($this->metadata['changed']) ||
--                  $this->metadata['objectcount'] === null ||
--                  $this->metadata['changed'] < date(self::DB_DATE_FORMAT, time() - $this->cache_refresh) ||
--                  $this->metadata['ctag'] != $this->folder->get_ctag() ||
--                  intval($this->metadata['objectcount']) !== $this->count()
-+            else if (
-+                empty($this->metadata['ctag'])
-+                || empty($this->metadata['changed'])
-+                || $this->metadata['ctag'] !== $ctag
-             ) {
-                 // lock synchronization for this folder or wait if locked
-                 $this->_sync_lock();
- 
--                // disable messages cache if configured to do so
--                $this->imap_mode(true);
-+                // Run a full-sync (initial sync or continue the aborted sync)
-+                if (empty($this->metadata['changed']) || empty($this->metadata['ctag'])) {
-+                    $result = $this->synchronize_full();
-+                }
-+                // Synchronize only the changes since last sync
-+                else {
-+                    $result = $this->synchronize_update($ctag);
-+                }
-+
-+                // update ctag value (will be written to database in _sync_unlock())
-+                if ($result) {
-+                    $this->metadata['ctag']    = $ctag;
-+                    $this->metadata['changed'] = date(self::DB_DATE_FORMAT, time());
-+                }
-+
-+                // remove lock
-+                $this->_sync_unlock();
-+            }
-+        }
- 
--                // synchronize IMAP mailbox cache
--                $this->imap->folder_sync($this->folder->name);
-+        $this->check_error();
-+        $this->synched = time();
-+    }
- 
--                // compare IMAP index with object cache index
--                $imap_index = $this->imap->index($this->folder->name, null, null, true, true);
-+    /**
-+     * Perform full cache synchronization
-+     */
-+    protected function synchronize_full()
-+    {
-+        // get effective time limit we have for synchronization (~70% of the execution time)
-+        $time_limit = $this->_max_sync_lock_time() * 0.7;
- 
--                $this->imap_mode(false);
-+        if (time() - $this->sync_start > $time_limit) {
-+            return false;
-+        }
- 
--                // determine objects to fetch or to invalidate
--                if (!$imap_index->is_error()) {
--                    $imap_index = $imap_index->get();
--                    $old_index  = array();
--                    $del_index  = array();
-+        // disable messages cache if configured to do so
-+        $this->imap_mode(true);
- 
--                    // read cache index
--                    $sql_result = $this->db->query(
--                        "SELECT `msguid`, `uid` FROM `{$this->cache_table}` WHERE `folder_id` = ?"
--                            . " ORDER BY `msguid` DESC", $this->folder_id
--                    );
-+        // synchronize IMAP mailbox cache, does nothing if messages cache is disabled
-+        $this->imap->folder_sync($this->folder->name);
- 
--                    while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
--                        // Mark all duplicates for removal (note sorting order above)
--                        // Duplicates here should not happen, but they do sometimes
--                        if (isset($old_index[$sql_arr['uid']])) {
--                            $del_index[] = $sql_arr['msguid'];
--                        }
--                        else {
--                            $old_index[$sql_arr['uid']] = $sql_arr['msguid'];
--                        }
--                    }
-+        // compare IMAP index with object cache index
-+        $imap_index = $this->imap->index($this->folder->name, null, null, true, true);
- 
--                    // fetch new objects from imap
--                    $i = 0;
--                    foreach (array_diff($imap_index, $old_index) as $msguid) {
--                        // Note: We'll store only objects matching the folder type
--                        // anything else will be silently ignored
--                        if ($object = $this->folder->read_object($msguid)) {
--                            // Deduplication: remove older objects with the same UID
--                            // Here we do not resolve conflicts, we just make sure
--                            // the most recent version of the object will be used
--                            if ($old_msguid = $old_index[$object['uid']]) {
--                                if ($old_msguid < $msguid) {
--                                    $del_index[] = $old_msguid;
--                                }
--                                else {
--                                    $del_index[] = $msguid;
--                                    continue;
--                                }
--                            }
--
--                            $old_index[$object['uid']] = $msguid;
--
--                            $this->_extended_insert($msguid, $object);
--
--                            // check time limit and abort sync if running too long
--                            if (++$i % 50 == 0 && time() - $sync_start > $time_limit) {
--                                $this->sync_complete = false;
--                                break;
--                            }
--                        }
--                    }
--                    $this->_extended_insert(0, null);
-+        $this->imap_mode(false);
- 
--                    $del_index = array_unique($del_index);
-+        if ($imap_index->is_error()) {
-+            rcube::raise_error(array(
-+                    'code' => 900,
-+                    'message' => "Failed to sync the kolab cache (SEARCH failed)"
-+                ), true);
-+            return false;
-+        }
- 
--                    // delete duplicate entries from IMAP
--                    $rem_index = array_intersect($del_index, $imap_index);
--                    if (!empty($rem_index)) {
--                        $this->imap_mode(true);
--                        $this->imap->delete_message($rem_index, $this->folder->name);
--                        $this->imap_mode(false);
--                    }
-+        // determine objects to fetch or to invalidate
-+        $imap_index = $imap_index->get();
-+        $del_index  = array();
-+        $old_index  = $this->current_index($del_index);
- 
--                    // delete old/invalid entries from the cache
--                    $del_index += array_diff($old_index, $imap_index);
--                    if (!empty($del_index)) {
--                        $quoted_ids = join(',', array_map(array($this->db, 'quote'), $del_index));
--                        $this->db->query(
--                            "DELETE FROM `{$this->cache_table}` WHERE `folder_id` = ? AND `msguid` IN ($quoted_ids)",
--                            $this->folder_id
--                        );
--                    }
-+        // Fetch objects and store in DB
-+        $result = $this->synchronize_fetch($imap_index, $old_index, $del_index);
- 
--                    // update ctag value (will be written to database in _sync_unlock())
--                    if ($this->sync_complete) {
--                        $this->metadata['ctag'] = $this->folder->get_ctag();
--                        $this->metadata['changed'] = date(self::DB_DATE_FORMAT, time());
--                        // remember the number of cache entries linked to this folder
--                        $this->metadata['objectcount'] = $this->count();
-+        if ($result) {
-+            // Remove redundant entries from IMAP and cache
-+            $rem_index = array_intersect($del_index, $imap_index);
-+            $del_index = array_merge(array_unique($del_index), array_diff($old_index, $imap_index));
-+
-+            $this->synchronize_delete($rem_index, $del_index);
-+        }
-+
-+        return $result;
-+    }
-+
-+    /**
-+     * Perform partial cache synchronization, based on QRESYNC
-+     */
-+    protected function synchronize_update()
-+    {
-+        if (!$this->imap->get_capability('QRESYNC')) {
-+            rcube::raise_error(array(
-+                    'code' => 900,
-+                    'message' => "Failed to sync the kolab cache (no QRESYNC capability)"
-+                ), true);
-+
-+            return $this->synchronize_full();
-+        }
-+
-+        // Handle the previous ctag
-+        list($uidvalidity, $highestmodseq, $uidnext) = explode('-', $this->metadata['ctag']);
-+
-+        if (empty($uidvalidity) || empty($highestmodseq)) {
-+            rcube::raise_error(array(
-+                    'code' => 900,
-+                    'message' => "Failed to sync the kolab cache (Invalid old ctag)"
-+                ), true);
-+            return false;
-+        }
-+
-+        // Enable QRESYNC
-+        $res = $this->imap->conn->enable('QRESYNC');
-+        if ($res === false) {
-+            rcube::raise_error(array(
-+                    'code' => 900,
-+                    'message' => "Failed to sync the kolab cache (failed to enable QRESYNC/CONDSTORE)"
-+                ), true);
-+
-+            return false;
-+        }
-+
-+        $mbox_data = $this->imap->folder_data($this->folder->name);
-+        if (empty($mbox_data)) {
-+            rcube::raise_error(array(
-+                    'code' => 900,
-+                    'message' => "Failed to sync the kolab cache (failed to get folder state)"
-+                ), true);
-+
-+             return false;
-+        }
-+
-+        // Check UIDVALIDITY
-+        if ($uidvalidity != $mbox_data['UIDVALIDITY']) {
-+            return $this->synchronize_full();
-+        }
-+
-+        // QRESYNC not supported on specified mailbox
-+        if (!empty($mbox_data['NOMODSEQ']) || empty($mbox_data['HIGHESTMODSEQ'])) {
-+            rcube::raise_error(array(
-+                    'code' => 900,
-+                    'message' => "Failed to sync the kolab cache (QRESYNC not supported on the folder)"
-+                ), true);
-+
-+             return $this->synchronize_full();
-+        }
-+
-+        // Get modified flags and vanished messages
-+        // UID FETCH 1:* (FLAGS) (CHANGEDSINCE 0123456789 VANISHED)
-+        $result = $this->imap->conn->fetch(
-+            $this->folder->name, '1:*', true, array('FLAGS'), $highestmodseq, true
-+        );
-+
-+        $removed  = array();
-+        $modified = array();
-+        $existing = $this->current_index($removed);
-+
-+        if (!empty($result)) {
-+            foreach ($result as $msg) {
-+                $uid = $msg->uid;
-+
-+                // Message marked as deleted
-+                if (!empty($msg->flags['DELETED'])) {
-+                    $removed[] = $uid;
-+                    continue;
-+                }
-+
-+                // Flags changed or new
-+                $modified[] = $uid;
-+            }
-+        }
-+
-+        $new    = array_diff($modified, $existing, $removed);
-+        $result = true;
-+
-+        if (!empty($new)) {
-+            $result = $this->synchronize_fetch($new, $existing, $removed);
-+
-+            if (!$result) {
-+                return false;
-+            }
-+        }
-+
-+        // VANISHED found?
-+        $mbox_data = $this->imap->folder_data($this->folder->name);
-+
-+        // Removed vanished messages from the database
-+        $vanished = (array) rcube_imap_generic::uncompressMessageSet($mbox_data['VANISHED']);
-+
-+        // Remove redundant entries from IMAP and DB
-+        $vanished = array_merge($removed, array_intersect($vanished, $existing));
-+        $this->synchronize_delete($removed, $vanished);
-+
-+        return $result;
-+    }
-+
-+    /**
-+     * Fetch objects from IMAP and save into the database
-+     */
-+    protected function synchronize_fetch($new_index, &$old_index, &$del_index)
-+    {
-+        // get effective time limit we have for synchronization (~70% of the execution time)
-+        $time_limit = $this->_max_sync_lock_time() * 0.7;
-+
-+        if (time() - $this->sync_start > $time_limit) {
-+            return false;
-+        }
-+
-+        $i = 0;
-+        $aborted = false;
-+
-+        // fetch new objects from imap
-+        foreach (array_diff($new_index, $old_index) as $msguid) {
-+            // Note: We'll store only objects matching the folder type
-+            // anything else will be silently ignored
-+            if ($object = $this->folder->read_object($msguid)) {
-+                // Deduplication: remove older objects with the same UID
-+                // Here we do not resolve conflicts, we just make sure
-+                // the most recent version of the object will be used
-+                if ($old_msguid = $old_index[$object['uid']]) {
-+                    if ($old_msguid < $msguid) {
-+                        $del_index[] = $old_msguid;
-+                    }
-+                    else {
-+                        $del_index[] = $msguid;
-+                        continue;
-                     }
-                 }
- 
--                // remove lock
--                $this->_sync_unlock();
-+                $old_index[$object['uid']] = $msguid;
-+
-+                $this->_extended_insert($msguid, $object);
-+
-+                // check time limit and abort sync if running too long
-+                if (++$i % 50 == 0 && time() - $this->sync_start > $time_limit) {
-+                    $aborted = true;
-+                    break;
-+                }
-             }
-         }
- 
--        $this->check_error();
--        $this->synched = time();
-+        $this->_extended_insert(0, null);
-+
-+        return $aborted === false;
-     }
- 
-+    /**
-+     * Remove specified objects from the database and IMAP
-+     */
-+    protected function synchronize_delete($imap_delete, $db_delete)
-+    {
-+        if (!empty($imap_delete)) {
-+            $this->imap_mode(true);
-+            $this->imap->delete_message($imap_delete, $this->folder->name);
-+            $this->imap_mode(false);
-+        }
-+
-+        if (!empty($db_delete)) {
-+            $quoted_ids = join(',', array_map(array($this->db, 'quote'), $db_delete));
-+            $this->db->query(
-+                "DELETE FROM `{$this->cache_table}` WHERE `folder_id` = ? AND `msguid` IN ($quoted_ids)",
-+                $this->folder_id
-+            );
-+        }
-+    }
-+
-+    /**
-+     * Return current use->msguid index
-+     */
-+    protected function current_index(&$duplicates = array())
-+    {
-+        // read cache index
-+        $sql_result = $this->db->query(
-+            "SELECT `msguid`, `uid` FROM `{$this->cache_table}` WHERE `folder_id` = ?"
-+                . " ORDER BY `msguid` DESC", $this->folder_id
-+        );
-+
-+        $index = $del_index = array();
-+
-+        while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
-+            // Mark all duplicates for removal (note sorting order above)
-+            // Duplicates here should not happen, but they do sometimes
-+            if (isset($index[$sql_arr['uid']])) {
-+                $duplicates[] = $sql_arr['msguid'];
-+            }
-+            else {
-+                $index[$sql_arr['uid']] = $sql_arr['msguid'];
-+            }
-+        }
-+
-+        return $index;
-+    }
- 
-     /**
-      * Read a single entry from cache or from IMAP directly
-@@ -1044,7 +1218,7 @@ class kolab_storage_cache
-             return;
- 
-         $sql_arr = $this->db->fetch_assoc($this->db->query(
--                "SELECT `folder_id`, `synclock`, `ctag`, `changed`, `objectcount`"
-+                "SELECT `folder_id`, `synclock`, `ctag`, `changed`"
-                 . " FROM `{$this->folders_table}` WHERE `resource` = ?",
-                 $this->resource_uri
-         ));
-@@ -1082,10 +1256,12 @@ class kolab_storage_cache
-         $read_query  = "SELECT `synclock`, `ctag` FROM `{$this->folders_table}` WHERE `folder_id` = ?";
-         $write_query = "UPDATE `{$this->folders_table}` SET `synclock` = ? WHERE `folder_id` = ? AND `synclock` = ?";
- 
-+        $max_lock_time = $this->_max_sync_lock_time();
-+
-         // wait if locked (expire locks after 10 minutes) ...
-         // ... or if setting lock fails (another process meanwhile set it)
-         while (
--            (intval($this->metadata['synclock']) + $this->max_sync_lock_time > time()) ||
-+            (intval($this->metadata['synclock']) + $max_lock_time > time()) ||
-             (($res = $this->db->query($write_query, time(), $this->folder_id, intval($this->metadata['synclock']))) &&
-                 !($affected = $this->db->affected_rows($res)))
-         ) {
-@@ -1105,16 +1281,26 @@ class kolab_storage_cache
-             return;
- 
-         $this->db->query(
--            "UPDATE `{$this->folders_table}` SET `synclock` = 0, `ctag` = ?, `changed` = ?, `objectcount` = ? WHERE `folder_id` = ?",
-+            "UPDATE `{$this->folders_table}` SET `synclock` = 0, `ctag` = ?, `changed` = ? WHERE `folder_id` = ?",
-             $this->metadata['ctag'],
-             $this->metadata['changed'],
--            $this->metadata['objectcount'],
-             $this->folder_id
-         );
- 
-         $this->synclock = false;
-     }
- 
-+    protected function _max_sync_lock_time()
-+    {
-+        $limit = get_offset_sec(ini_get('max_execution_time'));
-+
-+        if ($limit <= 0 || $limit > $this->max_sync_lock_time) {
-+            $limit = $this->max_sync_lock_time;
-+        }
-+
-+        return $limit;
-+    }
-+
-     /**
-      * Check IMAP connection error state
-      */
--- 
-2.25.4
-

debian.series Changed

Build Results
RPM Lint

Refresh

Comments for request 2793 0

Request History

vanmeeuwen created request about 3 years ago

Without patch

vanmeeuwen accepted review about 3 years ago

vanmeeuwen approved review about 3 years ago

vanmeeuwen accepted request about 3 years ago