Kolab Groupware OBS

We truncated the diff of some files because they were too big. If you want to see the full diff for every file, click here.

Changes of Revision 7

bonnie-0.3.6.tar.gz/. Added

bonnie-0.3.6.tar.gz/.gitignore Added

bonnie-0.3.6.tar.gz/INSTALL Added

@@ -0,0 +1,65 @@
+Installation Instructions for Enterprise Linux 6
+================================================
+
+.. WARNING::
+
+    These are **very** quick and **very** dirty installation instructions.
+
+#.  cat > /etc/yum.repos.d/elasticsearch-1.1.repo<<EOF
+[elasticsearch-1.1]
+name=Elasticsearch repository for 1.1.x packages
+baseurl=http://packages.elasticsearch.org/elasticsearch/1.1/centos
+gpgcheck=1
+gpgkey=http://packages.elasticsearch.org/GPG-KEY-elasticsearch
+enabled=1
+EOF
+
+#.  cat > /etc/yum.repos.d/home:vanmeeuwen:lesb.repo<<EOF
+[home_vanmeeuwen_lesb]
+name=Logstash, Elastic Search and Bonnie packages (CentOS_6)
+type=rpm-md
+baseurl=http://obs.kolabsys.com:82//home:/vanmeeuwen:/lesb/CentOS_6/
+gpgcheck=1
+gpgkey=http://obs.kolabsys.com:82//home:/vanmeeuwen:/lesb/CentOS_6/repodata/repomd.xml.key
+enabled=1
+EOF
+
+#.  yum install \
+        elasticsearch \
+        python-elasticsearch \
+        python-nosetests \
+        python-zmq \
+        python-twisted-core
+
+#.  chmod 755 /root/
+
+#.  ln -s $(pwd)/conf /etc/bonnie
+
+#.  cat>>/etc/imapd.conf<<EOF
+debug: 1
+iolog: 1
+auditlog: 1
+chatty: 1
+
+event_content_inclusion_mode: standard
+event_content_size: 0
+event_extra_params: bodyStructure clientAddress diskUsed flagNames messageSize messages modseq service timestamp uidnext vnd.cmu.midset vnd.cmu.unseenMessages vnd.cmu.envelope vnd.fastmail.clientId vnd.fastmail.sessionId vnd.fastmail.cid vnd.cmu.mbtype
+event_groups: message quota flags access mailbox
+event_notifier: external
+sievenotifier: external
+notify_external: /root/bonnie.git/dealer.py
+EOF
+
+#.  service cyrus-imapd restart
+
+#.  chkconfig --add elasticsearch
+
+#.  service elasticsearch start
+
+#.  cd /root/bonnie.git
+
+#.  ./broker.py &
+
+#.  ./worker.py &
+
+#.  ./collector.py &

bonnie-0.3.6.tar.gz/bonnie Added

bonnie-0.3.6.tar.gz/bonnie/__init__.py Added

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging
+import threading
+
+from bonnie.logger import Logger
+logging.setLoggerClass(Logger)
+
+API_VERSION = 1
+
+def getLogger(name):
+    """
+        Return the correct logger class.
+    """
+    logging.setLoggerClass(Logger)
+
+    log = logging.getLogger(name=name.replace(".", "_"))
+    return log
+
+log = getLogger('bonnie')
+
+from bonnie.conf import Conf
+conf = Conf()
+
+def getConf():
+    _data = threading.local()
+    if hasattr(_data, 'conf'):
+        log.debug(_("Returning thread local configuration"))
+        return _data.conf
+
+    return conf
+
+

bonnie-0.3.6.tar.gz/bonnie/broker Added

bonnie-0.3.6.tar.gz/bonnie/broker/__init__.py Added

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    This is the broker for Bonnie.
+"""
+import brokers
+
+from bonnie.daemon import BonnieDaemon
+
+class BonnieBroker(BonnieDaemon):
+    """
+
+    """
+    pidfile = "/var/run/bonnie/broker.pid"
+
+    def __init__(self, *args, **kw):
+        super(BonnieBroker, self).__init__(*args, **kw)
+
+        #: Interests are registered here.
+        self.broker_interests = {}
+
+        self.broker_modules = {}
+
+        for _class in brokers.list_classes():
+            module = _class()
+            module.register(callback=self.register_broker)
+            self.broker_modules[_class] = module
+
+    def register_broker(self, interests):
+        """
+            Register a broker based on interests
+        """
+
+        for interest,how in interests.iteritems():
+            if not self.broker_interests.has_key(interest):
+                self.broker_interests[interest] = []
+
+            self.broker_interests[interest].append(how)
+
+    def run(self, *args, **kw):
+        for interest, hows in self.broker_interests.iteritems():
+            for how in hows:
+                how()
+
+    def terminate(self, *args, **kw):
+        for module in self.broker_modules.values():
+            if hasattr(module, 'terminate'):
+                module.terminate()
+            else:
+                module.running = False

bonnie-0.3.6.tar.gz/bonnie/broker/brokers Added

bonnie-0.3.6.tar.gz/bonnie/broker/brokers/__init__.py Added

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from zmq_broker import ZMQBroker
+
+__all__ = [
+        'ZMQBroker'
+    ]
+
+def list_classes():
+    return [
+            ZMQBroker
+        ]

bonnie-0.3.6.tar.gz/bonnie/broker/brokers/zmq_broker Added

bonnie-0.3.6.tar.gz/bonnie/broker/brokers/zmq_broker/__init__.py Added

@@ -0,0 +1,618 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    This is the ZMQ broker implementation for Bonnie.
+"""
+
+import datetime
+from dateutil.parser import parse
+from dateutil.tz import tzutc
+import json
+from multiprocessing import Process
+import os
+import random
+import re
+import signal
+import sys
+import threading
+import time
+import zmq
+from zmq.eventloop import ioloop, zmqstream
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.broker.ZMQBroker')
+
+from bonnie.broker.state import init_db
+
+import collector
+import job
+import worker
+
+class ZMQBroker(object):
+    running = False
+
+    def __init__(self):
+        """
+            A ZMQ Broker for Bonnie.
+        """
+        self.context = zmq.Context(io_threads=128)
+        self.poller = zmq.Poller()
+
+        self.routers = {}
+        self.router_processes = {}
+
+        self.collector_jobs_pending = 0
+
+    def register(self, callback):
+        callback({ '_all': self.run })
+
+    def create_router(self, name, default, **kw):
+        """
+            Create a regular router that remains
+            available for the base ZMQBroker.
+        """
+        bind_address = conf.get(
+                'broker',
+                'zmq_%s_router_bind_address' % (name)
+            )
+
+        if bind_address == None:
+            bind_address = default
+
+        router = self.context.socket(zmq.ROUTER)
+        router.bind(bind_address)
+
+        self.routers[name] = {
+                'router': router,
+                'callbacks': dict(kw)
+            }
+
+        self.poller.register(router, zmq.POLLIN)
+
+        return router
+
+    def create_router_process(self, name, default, callback):
+        """
+            Create a router process (that is no
+            longer available to the base ZMQBroker).
+        """
+
+        bind_address = conf.get(
+                'broker',
+                'zmq_%s_router_bind_address' % (name)
+            )
+
+        if bind_address == None:
+            bind_address = default
+
+        self.router_processes[name] = Process(
+                target=self._run_router_process,
+                args=(callback, bind_address)
+            )
+
+        self.router_processes[name].start()
+
+    def find_router(self, socket):
+        for name, attrs in self.routers.iteritems():
+            if attrs['router'] == socket:
+                return name, attrs['router'], attrs['callbacks']
+
+    def run(self):
+        self.create_router(
+                'collector',
+                'tcp://*:5571',
+                recv_multipart = self._cb_cr_recv_multipart
+            )
+
+        self.create_router_process(
+                'dealer',
+                'tcp://*:5570',
+                self._cb_dr_on_recv_stream
+            )
+
+        self.create_router(
+                'worker',
+                'tcp://*:5573',
+                recv_multipart = self._cb_wr_recv_multipart
+            )
+
+        self.create_router(
+                'worker_controller',
+                'tcp://*:5572',
+                recv_multipart = self._cb_wcr_recv_multipart
+            )
+
+        self.running = True
+        last_expire = time.time()
+        last_state = time.time()
+        last_vacuum = time.time()
+
+        try:
+            db = init_db('broker')
+            db.execute("VACUUM")
+            db.commit()
+        except Exception, errmsg:
+            pass
+
+        poller_timeout = int(conf.get('broker', 'zmq_poller_timeout', 100))
+
+        while self.running:
+            try:
+                # TODO: adjust polling timout according to the number
+                # of pending jobs.
+                sockets = dict(self.poller.poll(poller_timeout))
+
+            except KeyboardInterrupt, errmsg:
+                log.info("zmq.Poller KeyboardInterrupt")
+                break
+
+            except Exception, errmsg:
+                log.error("zmq.Poller error: %r", errmsg)
+                sockets = dict()
+
+            for socket, event in sockets.iteritems():
+                if event == zmq.POLLIN:
+                    name, router, callbacks = self.find_router(socket)
+
+                    for callforward, callback in callbacks.iteritems():
+                        result = getattr(router, '%s' % callforward)()
+                        callback(router, result)
+
+            # Once every 30 seconds, expire stale collectors and
+            # workers, unlock jobs and expire those done.
+            if last_expire < (time.time() - 30):
+                collector.expire()
+                worker.expire()
+                job.unlock()
+                job.expire()
+
+                last_expire = time.time()
+
+            # Report on the state of jobs.
+            if last_state < (time.time() - 30):
+                self._write_stats()
+                last_state = time.time()
+
+        log.info("Shutting down")
+
+        for attrs in self.routers.values():
+            attrs['router'].close()
+

bonnie-0.3.6.tar.gz/bonnie/broker/brokers/zmq_broker/collector.py Added

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from bonnie.broker.state import init_db, Collector, Job, Interest
+
+import datetime
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.broker.ZMQBroker')
+
+
+def add(identity, state=b'READY', interests=[]):
+    db = init_db('collectors')
+    db.add(Collector(identity, state))
+    db.commit()
+    collector = db.query(Collector).filter_by(identity=identity).first()
+    for cmd in interests:
+        interest = db.query(Interest).filter_by(cmd=cmd).first()
+
+        if interest is None:
+            db.add(Interest(cmd))
+            db.commit()
+            interest = db.query(Interest).filter_by(cmd=cmd).first()
+
+        collector.interests.append(interest)
+
+    db.commit()
+
+
+def count():
+    db = init_db('collectors')
+    result = db.query(Collector).count()
+    return result
+
+
+def count_by_state(state):
+    db = init_db('collectors')
+    result = db.query(Collector).filter_by(state=state).count()
+    return result
+
+
+def select(identity):
+    db = init_db('collectors')
+    collector = db.query(Collector).filter_by(identity=identity).first()
+    return collector
+
+
+def select_by_state(state):
+    db = init_db('collectors')
+    collectors = db.query(Collector).filter_by(state=state).all()
+    return collectors
+
+
+def set_state(identity, state, interests=[]):
+    db = init_db('collectors')
+
+    collector = db.query(Collector).filter_by(identity=identity).first()
+
+    if collector is None:
+        db.add(Collector(identity, state))
+        db.commit()
+        collector = db.query(Collector).filter_by(identity=identity).first()
+    else:
+        collector.state = state
+        collector.timestamp = datetime.datetime.utcnow()
+        db.commit()
+
+    if state == b'READY':
+        collector.job = None
+
+    for cmd in interests:
+        interest = db.query(Interest).filter_by(cmd=cmd).first()
+
+        if interest is None:
+            db.add(Interest(cmd))
+            db.commit()
+            interest = db.query(Interest).filter_by(cmd=cmd).first()
+
+        collector.interests.append(interest)
+
+    db.commit()
+
+
+def update(identity, **kw):
+    db = init_db('collectors')
+
+    collector = db.query(Collector).filter_by(identity=identity).first()
+
+    if collector is None:
+        db.add(Collector(identity))
+        db.commit()
+        collector = db.query(Collector).filter_by(identity=identity).first()
+
+    for attr, value in kw.iteritems():
+        setattr(collector, attr, value)
+
+    db.commit()
+
+
+def expire():
+    db = init_db('collectors')
+
+    for collector in db.query(Collector).filter(Collector.timestamp <= (datetime.datetime.utcnow() - datetime.timedelta(0, 90)), Collector.state == b'STALE').all():
+        log.debug("Purging collector %s as very stale" % (collector.identity), level=7)
+        for _job in db.query(Job).filter_by(collector=collector.identity).all():
+            _job.state = b'PENDING'
+            _job.timestamp = datetime.datetime.utcnow()
+
+        db.delete(collector)
+        db.commit()
+
+    for collector in db.query(Collector).filter(Collector.timestamp <= (datetime.datetime.utcnow() - datetime.timedelta(0, 90)), Collector.state != b'STALE').all():
+        log.debug("Marking collector %s as stale" % (collector.identity), level=7)
+        for _job in db.query(Job).filter_by(collector=collector.identity).all():
+            _job.state = b'PENDING'
+            _job.timestamp = datetime.datetime.utcnow()
+
+        collector.state = b'STALE'
+        collector.timestamp = datetime.datetime.utcnow()
+        db.commit()

bonnie-0.3.6.tar.gz/bonnie/broker/brokers/zmq_broker/job.py Added

@@ -0,0 +1,302 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from bonnie.broker.state import init_db, Collector, Job, Worker
+
+import datetime
+import time
+
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.exc import OperationalError
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.broker.ZMQBroker')
+
+MAX_RETRIES = 30
+
+
+def add(dealer, notification, job_type='worker'):
+    """
+        Add a new job.
+    """
+    db = init_db('jobs')
+
+    if db is None:
+        return None
+
+    job = None
+
+    try:
+        job = Job(dealer, notification, job_type)
+        db.add(job)
+        db.commit()
+    except IntegrityError, errmsg:
+        log.error("SQLAlchemy Integrity Error: %r" % (errmsg))
+        db.rollback()
+        return None
+    except OperationalError, errmsg:
+        log.error("SQLAlchemy Operational Error: %r" % (errmsg))
+        db.rollback()
+        return None
+
+    return job
+
+
+def count():
+    db = init_db('jobs')
+    result = db.query(Job).count()
+    return result
+
+
+def count_by_state(state):
+    db = init_db('jobs')
+    result = db.query(Job).filter_by(state=state).count()
+    return result
+
+
+def count_by_type(job_type):
+    db = init_db('jobs')
+    result = db.query(Job).filter_by(job_type=job_type).all()
+    return result
+
+
+def count_by_type_and_state(job_type, state):
+    db = init_db('jobs')
+    result = db.query(Job).filter_by(job_type=job_type, state=state).count()
+    return result
+
+
+def first():
+    db = init_db('jobs')
+    result = db.query(
+        Job
+    ).filter(
+        Job.state != b'DONE',
+        Job.state != b'FAILED'
+    ).order_by(
+        Job.id
+    ).first()
+
+    return result
+
+
+def select(job_uuid):
+    db = init_db('jobs')
+    result = db.query(Job).filter_by(uuid=job_uuid).first()
+    return result
+
+
+def select_all():
+    db = init_db('jobs')
+    result = db.query(Job).all()
+    return result
+
+
+def select_by_state(state):
+    db = init_db('jobs')
+    result = db.query(Job).filter_by(state=state).all()
+    return result
+
+
+def select_by_type(job_type):
+    db = init_db('jobs')
+    result = db.query(Job).filter_by(job_type=job_type).all()
+    return result
+
+
+def select_for_collector(identity):
+    db = init_db('jobs')
+
+    collector = db.query(
+        Collector
+    ).filter(
+        Collector.identity == identity,
+        Collector.job == None,
+        Collector.state == b'READY'
+    ).first()
+
+    if collector is None:
+        return
+
+    # This is influenced by .update(), which resets the .timestamp to
+    # .utcnow(), effectively pushing all updated jobs to the back of the
+    # queue.
+    #
+    # Practical result is a massive amount of metadata gathering
+    # followed by a sudden surge of jobs getting DONE.
+    # job = db.query(Job).filter_by(collector=identity, job_type='collector',
+    #       state=b'PENDING').order_by(Job.timestamp).first()
+
+    # This would result in "most recent first, work your way backwards."
+    # job = db.query(Job).filter_by(collector=identity, job_type='collector',
+    #       state=b'PENDING').order_by(Job.timestamp.desc()).first()
+
+    # This results in following the storage order and is by far the
+    # fastest methodology.
+    job = db.query(
+        Job
+    ).filter_by(
+        collector=identity,
+        job_type='collector',
+        state=b'PENDING'
+    ).order_by(
+        Job.id
+    ).first()
+
+    if job == None:
+        return
+
+    job.state = b'ALLOC'
+    job.timestamp = datetime.datetime.utcnow()
+    db.commit()
+
+    return job
+
+
+def select_for_worker(identity):
+    db = init_db('jobs')
+
+    worker = db.query(Worker).filter(Worker.identity == identity, Worker.job == None, Worker.state == b'READY').first()
+
+    if worker == None:
+        return
+
+    # This is influenced by .update(), which resets the .timestamp to
+    # .utcnow(), effectively pushing all updated jobs to the back of the
+    # queue.
+    #
+    # Practical result is a massive amount of metadata gathering
+    # followed by a sudden surge of jobs getting DONE.
+    #job = db.query(Job).filter_by(job_type='worker', state=b'PENDING').order_by(Job.timestamp).first()
+
+    # This would result in "most recent first, work your way backwards."
+    #job = db.query(Job).filter_by(job_type='worker', state=b'PENDING').order_by(Job.timestamp.desc()).first()
+
+    # This results in following the storage order and is by far the
+    # fastest methodology.
+    job = db.query(Job).filter_by(job_type='worker', state=b'PENDING').order_by(Job.id).first()
+
+    if job == None:
+        return
+

bonnie-0.3.6.tar.gz/bonnie/broker/brokers/zmq_broker/worker.py Added

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import datetime
+import time
+
+from bonnie.broker.state import init_db, Job, Worker
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.broker.ZMQBroker')
+
+def add(identity, state = b'READY'):
+    db = init_db('workers')
+    db.add(Worker(identity, state))
+    db.commit()
+
+def count():
+    db = init_db('workers')
+    result = db.query(Worker).count()
+    return result
+
+def count_by_state(state):
+    db = init_db('workers')
+    result = db.query(Worker).filter_by(state=state).count()
+    return result
+
+def select_by_state(state):
+    db = init_db('workers')
+    result = db.query(Worker).filter_by(state=state).all()
+    return result
+
+def select(identity):
+    db = init_db('workers')
+    result = db.query(Worker).filter_by(identity=identity).first()
+    return result
+
+def set_job(identity, job_uuid):
+    db = init_db('workers')
+    job = db.query(Job).filter_by(uuid=job_uuid).first()
+
+    if job == None:
+        return
+
+    worker = db.query(Worker).filter_by(identity=identity).first()
+    worker.job = job.id
+    job.worker = worker.id
+    db.commit()
+
+def set_state(identity, state):
+    db = init_db('workers')
+    worker = db.query(Worker).filter_by(identity=identity).first()
+
+    if worker == None:
+        db.add(Worker(identity, state))
+        db.commit()
+
+    else:
+        worker.state = state
+        worker.timestamp = datetime.datetime.utcnow()
+        db.commit()
+
+def update(identity, **kw):
+    db = init_db('identity')
+    worker = db.query(Worker).filter_by(identity=identity).first()
+
+    if worker == None:
+        db.add(Worker(identity, b'READY'))
+    else:
+        for attr, value in kw.iteritems():
+            setattr(worker, attr, value)
+
+        worker.timestamp = datetime.datetime.utcnow()
+
+    db.commit()
+
+def expire():
+    db = init_db('workers')
+    for worker in db.query(Worker).filter(Worker.timestamp <= (datetime.datetime.utcnow() - datetime.timedelta(0, 90)), Worker.state == b'STALE').all():
+        log.debug("Purging worker %s as very stale" % (worker.identity), level=7)
+
+        if not worker.job == None:
+            _job = db.query(Job).filter_by(id=worker.job).first()
+            if not _job == None:
+                _job.state = b'PENDING'
+                _job.timestamp = datetime.datetime.utcnow()
+
+        db.delete(worker)
+        db.commit()
+
+    for worker in db.query(Worker).filter(Worker.timestamp <= (datetime.datetime.utcnow() - datetime.timedelta(0, 90)), Worker.state != b'STALE').all():
+        log.debug("Marking worker %s as stale" % (worker.identity), level=7)
+        if not worker.job == None:
+            _job = db.query(Job).filter_by(id=worker.job).first()
+            if not _job == None:
+                _job.state = b'PENDING'
+                _job.timestamp = datetime.datetime.utcnow()
+
+        worker.state = b'STALE'
+        worker.timestamp = datetime.datetime.utcnow()
+        db.commit()

bonnie-0.3.6.tar.gz/bonnie/broker/state.py Added

@@ -0,0 +1,167 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import datetime
+import hashlib
+import re
+
+import sqlalchemy
+
+from sqlalchemy import Column
+from sqlalchemy import DateTime
+from sqlalchemy import ForeignKey
+from sqlalchemy import Integer
+from sqlalchemy import MetaData
+from sqlalchemy import String
+from sqlalchemy import Table
+from sqlalchemy import Text
+
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+from sqlalchemy.schema import Index
+
+try:
+    from sqlalchemy.orm import relationship
+except:
+    from sqlalchemy.orm import relation as relationship
+
+try:
+    from sqlalchemy.orm import sessionmaker
+except:
+    from sqlalchemy.orm import create_session
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.broker.state')
+
+DeclarativeBase = declarative_base()
+
+db = None
+
+collector_interest_table = Table(
+        'collector_interest',
+        DeclarativeBase.metadata,
+        Column('collector_id', Integer, ForeignKey('collector.id')),
+        Column('interest_id', Integer, ForeignKey('interest.id'))
+    )
+
+##
+## Classes
+##
+
+class Collector(DeclarativeBase):
+    __tablename__ = 'collector'
+
+    id = Column(Integer, primary_key=True)
+    identity = Column(String(128))
+    state = Column(String(16))
+    job = Column(Integer, ForeignKey('job.id'))
+    timestamp = Column(DateTime, default=datetime.datetime.utcnow())
+    interests = relationship(
+            "Interest",
+            secondary=collector_interest_table
+        )
+
+    def __init__(self, identity, state = b'READY'):
+        DeclarativeBase.__init__(self)
+        self.identity = identity
+        self.state = state
+
+class Interest(DeclarativeBase):
+    __tablename__ = 'interest'
+
+    id = Column(Integer, primary_key=True)
+    cmd = Column(String(16), nullable=False)
+
+    def __init__(self, cmd):
+        self.cmd = cmd
+
+class Job(DeclarativeBase):
+    __tablename__ = 'job'
+
+    id = Column(Integer, primary_key=True)
+    uuid = Column(String(128), nullable=False, unique=True)
+    dealer = Column(String(128), nullable=False)
+    collector = Column(String(128), nullable=False)
+    notification = Column(Text)
+    job_type = Column(String(16), default='worker')
+    state = Column(String(16))
+    cmd = Column(String(256))
+    timestamp = Column(DateTime, default=datetime.datetime.utcnow())
+    pushbacks = Column(Integer, default=0)
+
+    def __init__(self, dealer, notification, job_type='worker'):
+        DeclarativeBase.__init__(self)
+        self.uuid = hashlib.sha224(notification).hexdigest()
+        self.dealer = dealer
+        self.collector = re.sub(r'-[0-9]+$', '', dealer).replace('Dealer', 'Collector')
+        self.notification = notification
+        self.state = b'PENDING'
+        self.job_type = job_type
+        self.timestamp = datetime.datetime.utcnow()
+        self.cmd = None
+        self.pushbacks = 0
+
+Index('job_collector', 'collector')
+Index('job_job_type', 'job_type')
+Index('job_state', 'state')
+
+class Worker(DeclarativeBase):
+    __tablename__ = 'worker'
+
+    id = Column(Integer, primary_key=True)
+    identity = Column(String(128))
+    state = Column(String(16))
+    timestamp = Column(DateTime, default=datetime.datetime.utcnow())
+    job = Column(Integer, ForeignKey('job.id'))
+
+    def __init__(self, identity, state):
+        DeclarativeBase.__init__(self)
+        self.identity = identity
+        self.state = state
+
+def init_db(name):
+    """
+        Returns a SQLAlchemy Session() instance.
+    """
+    global db
+
+    if not db == None:
+        return db
+
+    db_uri = conf.get('broker', 'state_sql_uri')
+
+    if not db_uri:
+        db_uri = 'sqlite:////var/lib/bonnie/state.db'
+
+    echo = conf.debuglevel > 8
+
+    try:
+        engine = create_engine(db_uri, echo=echo)
+        DeclarativeBase.metadata.create_all(engine)
+    except Exception, errmsg:
+        log.error("Exception occurred: %r" % (errmsg))
+        return None
+
+    Session = sessionmaker(bind=engine,autoflush=True)
+    db = Session()
+
+    return db

bonnie-0.3.6.tar.gz/bonnie/collector Added

bonnie-0.3.6.tar.gz/bonnie/collector/__init__.py Added

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+import inputs
+import handlers
+import multiprocessing
+from distutils import version
+
+from bonnie.daemon import BonnieDaemon
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.collector')
+
+
+class BonnieCollector(BonnieDaemon):
+    pidfile = "/var/run/bonnie/collector.pid"
+
+    def __init__(self, *args, **kw):
+        super(BonnieCollector, self).__init__(*args, **kw)
+
+        self.input_interests = {}
+        self.input_modules = {}
+
+        self.handler_interests = {}
+
+        self.num_threads = int(conf.get('collector', 'num_threads', 5))
+        self.num_threads_busy = 0
+
+        sys_version = version.StrictVersion(sys.version[:3])
+
+        if sys_version >= version.StrictVersion("2.7"):
+            self.pool = multiprocessing.Pool(
+                self.num_threads,
+                self._worker_process_start,
+                (),
+                1
+            )
+
+        else:
+            self.pool = multiprocessing.Pool(
+                self.num_threads,
+                self._worker_process_start,
+                ()
+            )
+
+    def execute(self, commands, job_uuid, notification):
+        """
+            Dispatch collector job to the according handler(s)
+        """
+        self.num_threads_busy += 1
+
+        # execute this asynchronously in a child process
+        self.pool.apply_async(
+            async_execute_handlers,
+            (
+                commands.split(),
+                notification,
+                job_uuid
+            ),
+            callback=self._execute_callback
+        )
+
+    def register_input(self, interests):
+        self.input_interests = interests
+
+    def register_handler(self, interests={}):
+        for interest, how in interests.iteritems():
+            if interest not in self.handler_interests:
+                self.handler_interests[interest] = []
+
+            self.handler_interests[interest].append(how)
+
+    def run(self):
+        for _class in inputs.list_classes():
+            module = _class()
+            module.register(callback=self.register_input)
+            self.input_modules[_class] = module
+
+        for _class in handlers.list_classes():
+            handler = _class()
+            handler.register(callback=self.register_handler)
+
+        input_modules = conf.get('collector', 'input_modules')
+
+        if input_modules is None:
+            input_modules = ""
+
+        input_modules = [x.strip() for x in input_modules.split(',')]
+
+        for _input in self.input_modules.values():
+            if _input.name() in input_modules:
+                _input.run(
+                    callback=self.execute,
+                    interests=self.handler_interests.keys()
+                )
+
+    def terminate(self, *args, **kw):
+        for _input in self.input_modules.values():
+            if hasattr(_input, 'terminate'):
+                _input.terminate()
+            else:
+                _input.running = False
+
+        self.pool.close()
+
+    def _execute_callback(self, result):
+        (notification, job_uuid) = result
+
+        self.num_threads_busy -= 1
+
+        # pass result back to input module(s)
+        input_modules = conf.get('collector', 'input_modules').split(',')
+        for _input in self.input_modules.values():
+            if _input.name() in input_modules:
+                _input.callback_done(
+                    job_uuid,
+                    notification,
+                    threads=self._threads_available()
+                )
+
+    def _threads_available(self):
+        return (self.num_threads - self.num_threads_busy)
+
+    def _worker_process_start(self, *args, **kw):
+        log.info(
+            "Worker process %s initializing" % (
+                multiprocessing.current_process().name
+            )
+        )
+
+
+def async_execute_handlers(commands, notification, job_uuid):
+    """
+        Routine to execute handlers for the given commands and notification
+
+        To be run an an asynchronous child process.
+    """
+
+    log.info(
+        "COLLECT %r for %s by %s" % (
+            commands,
+            job_uuid,
+            multiprocessing.current_process().name
+        )
+    )
+
+    # register handlers with the interrests again in this subprocess
+    handler_interests = {}
+
+    def register_handler(interests={}):
+        for interest, how in interests.iteritems():
+            if interest not in handler_interests:
+                handler_interests[interest] = []
+
+            handler_interests[interest].append(how)
+
+    for _class in handlers.list_classes():
+        handler = _class()
+        handler.register(callback=register_handler)
+
+    log.debug(
+        "async_execute_handlers %r for job %r" % (commands, job_uuid),
+        level=8
+    )
+
+    for command in commands:
+        if command in handler_interests:
+            for interest in handler_interests[command]:
+                notification = interest['callback'](notification=notification)
+
+    return (notification, job_uuid)

bonnie-0.3.6.tar.gz/bonnie/collector/handlers Added

bonnie-0.3.6.tar.gz/bonnie/collector/handlers/__init__.py Added

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from messagedata import MessageDataHandler
+from imapdata import IMAPDataHandler
+from ldapdata import LDAPDataHandler
+
+__all__ = [
+    'MessageDataHandler',
+    'IMAPDataHandler',
+    'LDAPDataHandler'
+]
+
+def list_classes():
+    return [
+        MessageDataHandler,
+        IMAPDataHandler,
+        LDAPDataHandler
+    ]

bonnie-0.3.6.tar.gz/bonnie/collector/handlers/imapdata.py Added

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+import pykolab
+from pykolab.imap import IMAP
+
+import bonnie
+from bonnie.utils import parse_imap_uri
+from bonnie.utils import imap_folder_path
+
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.collector.IMAPDataHandler')
+
+
+class IMAPDataHandler(object):
+    """
+        Collector handler to provide metadata from IMAP
+    """
+
+    def __init__(self, *args, **kw):
+        # load pykolab conf
+        conf = pykolab.getConf()
+        if not hasattr(conf, 'defaults'):
+            conf.finalize_conf(fatal=False)
+
+        self.imap = IMAP()
+
+    def register(self, callback):
+        interests = {
+                'GETMETADATA': {
+                        'callback': self.get_imap_folder_metadata
+                    },
+                'GETACL': {
+                        'callback': self.get_imap_folder_acl
+                    }
+            }
+
+        callback(interests)
+
+    def get_imap_folder_metadata(self, notification):
+        notification = json.loads(notification)
+        log.debug("GETMETADATA for %r" % (notification), level=9)
+
+        # split the uri parameter into useful parts
+        uri = parse_imap_uri(notification['uri'])
+        folder_path = imap_folder_path(uri)
+
+        # get metadata using pykolab's imap module
+        metadata = {}
+        try:
+            self.imap.connect()
+            metadata = self.imap.get_metadata(folder_path)[folder_path]
+            self.imap.disconnect()
+        except Exception, e:
+            log.warning("Failed to get metadata for %r: %r", folder_path, e)
+
+        notification['metadata'] = metadata
+
+        return json.dumps(notification)
+
+    def get_imap_folder_acl(self, notification):
+        notification = json.loads(notification)
+        log.debug("GETACL for %r" % (notification), level=9)
+
+        # split the uri parameter into useful parts
+        uri = parse_imap_uri(notification['uri'])
+        folder_path = imap_folder_path(uri)
+
+        # get folder acls using pykolab's imap module
+        acls = {}
+        try:
+            self.imap.connect()
+            acls = self.imap.list_acls(folder_path)
+            self.imap.disconnect()
+        except Exception, e:
+            log.warning("Failed to get ACLs for %r: %r", folder_path, e)
+
+        notification['acl'] = acls
+
+        return json.dumps(notification)

bonnie-0.3.6.tar.gz/bonnie/collector/handlers/ldapdata.py Added

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+import pykolab
+from pykolab.auth import Auth
+
+import bonnie
+from bonnie.utils import parse_imap_uri
+from bonnie.utils import imap_folder_path
+
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.collector.LDAPDataHandler')
+
+class LDAPDataHandler(object):
+    """
+        Collector handler to provide user data from LDAP
+    """
+
+    def __init__(self, *args, **kw):
+        # load pykolab conf
+        self.pykolab_conf = pykolab.getConf()
+        if not hasattr(self.pykolab_conf, 'defaults'):
+            self.pykolab_conf.finalize_conf(fatal=False)
+
+        self.ldap = Auth()
+        self.ldap.connect()
+
+    def register(self, callback):
+        interests = {
+                'GETUSERDATA': { 'callback': self.get_user_data }
+            }
+
+        callback(interests)
+
+    def get_user_data(self, notification):
+        notification = json.loads(notification)
+        log.debug("GETUSERDATA for %r" % (notification), level=9)
+
+        if notification.has_key('user'):
+            try:
+                user_dn = self.ldap.find_user_dn(notification['user'], True)
+                log.debug("User DN for %s: %r" % (notification['user'], user_dn), level=8)
+            except Exception, e:
+                log.error("LDAP connection error: %r", e)
+                user_dn = None
+
+            if user_dn:
+                unique_attr = self.pykolab_conf.get('ldap', 'unique_attribute', 'nsuniqueid')
+                user_rec = self.ldap.get_entry_attributes(None, user_dn, [unique_attr, 'cn'])
+                log.debug("User attributes: %r" % (user_rec), level=8)
+
+                if user_rec and user_rec.has_key(unique_attr):
+                    user_rec['dn'] = user_dn
+                    user_rec['id'] = user_rec[unique_attr]
+                    del user_rec[unique_attr]
+            else:
+                user_rec = None
+
+            notification['user_data'] = user_rec
+
+        return json.dumps(notification)

bonnie-0.3.6.tar.gz/bonnie/collector/handlers/messagedata.py Added

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import time
+import json
+
+from email import message_from_string
+
+import bonnie
+from bonnie import utils
+from bonnie.utils import expand_uidset
+from bonnie.utils import parse_imap_uri
+from bonnie.utils import decode_message_headers
+
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.collector.MessageDataHandler')
+
+
+class MessageDataHandler(object):
+
+    def __init__(self, *args, **kw):
+        pass
+
+    def register(self, callback):
+        interests = {
+                'FETCH':  {
+                        'callback': self.retrieve_contents_for_messages
+                    },
+                'HEADER': {
+                        'callback': self.retrieve_headers_for_messages
+                    }
+            }
+
+        callback(interests)
+
+    def retrieve_contents_for_messages(self, notification):
+        notification = json.loads(notification)
+        log.debug("FETCH for %r" % (notification), level=9)
+
+        (
+            messageContents,
+            messageHeaders
+        ) = self._retrieve_contents_for_messages(notification)
+
+        # set new notification properties, even if empty
+        notification['messageContent'] = messageContents
+        notification['messageHeaders'] = messageHeaders
+
+        return json.dumps(notification)
+
+    def retrieve_headers_for_messages(self, notification):
+        notification = json.loads(notification)
+        log.debug("HEADERS for %r" % (notification), level=9)
+
+        (
+            messageContents,
+            messageHeaders
+        ) = self._retrieve_contents_for_messages(notification, True)
+
+        # set new notification properties, even if empty
+        notification['messageHeaders'] = messageHeaders
+
+        return json.dumps(notification)
+
+    def _retrieve_contents_for_messages(self, notification, headers_only=False):
+        """
+            Helper method to deliver message contents/headers
+        """
+        messageContents = {}
+        messageHeaders = {}
+
+        # split the uri parameter into useful parts
+        uri = parse_imap_uri(notification['uri'])
+
+        if notification.has_key('uidset'):
+            message_uids = expand_uidset(notification['uidset'])
+        elif notification.has_key('vnd.cmu.oldUidset'):
+            message_uids = expand_uidset(notification['vnd.cmu.oldUidset'])
+        elif uri.has_key('UID'):
+            message_uids = [ uri['UID'] ]
+            notification['uidset'] = ','.join(message_uids)
+
+        # resolve uri into a mailbox path on the local file stystem
+        mailbox_path = utils.imap_mailbox_fs_path(uri)
+
+        log.debug("Using mailbox path: %r" % (mailbox_path), level=8)
+
+        # mailbox exists, try reading the message files
+        if os.path.exists(mailbox_path):
+            for message_uid in message_uids:
+                # using message file path like /var/spool/imap/domain/e/example.org/k/lists/kolab^org/devel/lists/kolab.org/devel@example.org/1.
+                message_file_path = "%s/%s." % (mailbox_path, message_uid)
+
+                log.debug("Open message file: %r" % (message_file_path), level=8)
+
+                attempts = 5
+                while attempts > 0:
+                    attempts -= 1
+
+                    if os.access(message_file_path, os.R_OK):
+                        fp = open(message_file_path, 'r')
+
+                        if headers_only:
+                            data = ''
+                            for line in fp:
+                                data += line
+                                if line.strip() == '':
+                                    break;
+                            data += "\r\n"
+                        else:
+                            data = fp.read()
+
+                        fp.close()
+
+                        # use email lib to parse message headers
+                        try:
+                            # find header delimiter
+                            pos = data.find("\r\n\r\n")
+                            message = message_from_string(data[0:pos])
+                            headers = decode_message_headers(message)
+                        except:
+                            headers = dict()
+                            messageHeaders[message_uid] = headers
+
+                        # append raw message data and parsed headers
+                        messageContents[message_uid] = data
+                        messageHeaders[message_uid] = headers
+                        break
+
+                    elif attempts > 0:
+                        log.debug("Failed to open message file %r; retry %d more times" % (
+                            message_file_path, attempts
+                        ), level=5)
+                    else:
+                        log.warning("Failed to open message file %r for uri=%s; uid=%s" % (
+                            message_file_path, notification, message_uid
+                        ))
+
+                    time.sleep(1)
+                # end while
+            # end for
+        else:
+            log.warning("Mailbox path %r does not exits for uri=%s" % (mailbox_path, notification['uri']))
+
+        return (messageContents, messageHeaders)

bonnie-0.3.6.tar.gz/bonnie/collector/inputs Added

bonnie-0.3.6.tar.gz/bonnie/collector/inputs/__init__.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from zmq_input import ZMQInput
+
+__all__ = [
+        'ZMQInput'
+    ]
+
+def list_classes():
+    return [
+            ZMQInput
+        ]
+

bonnie-0.3.6.tar.gz/bonnie/collector/inputs/zmq_input.py Added

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+
+    This is a collector node pulling jobs from a ZMQ broker.
+
+"""
+
+import datetime
+import json
+import os
+import socket
+import time
+import zmq
+from zmq.eventloop import ioloop, zmqstream
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.collector.ZMQInput')
+
+class ZMQInput(object):
+    report_timestamp = 0
+    state = b"READY"
+
+    def __init__(self, *args, **kw):
+        self.interests = []
+        self.context = zmq.Context()
+
+        zmq_broker_address = conf.get('collector', 'zmq_broker_address')
+
+        if zmq_broker_address == None:
+            zmq_broker_address = "tcp://localhost:5571"
+
+        self.identity = u"Collector-%s" % (socket.getfqdn())
+        self.collector = self.context.socket(zmq.DEALER)
+        self.collector.identity = (self.identity).encode('ascii')
+        self.collector.connect(zmq_broker_address)
+
+        self.stream = zmqstream.ZMQStream(self.collector)
+        self.ioloop = ioloop.IOLoop.instance()
+
+    def name(self):
+        return 'zmq_input'
+
+    def register(self, *args, **kw):
+        pass
+
+    def report_state(self, new_timeout=False):
+        log.debug("[%s] Reporting state %s, %r" % (self.identity, self.state, self.interests), level=9)
+
+        if self.report_timestamp < (time.time() - 10):
+            self.collector.send_multipart([b"STATE", self.state, " ".join(self.interests)])
+            self.report_timestamp = time.time()
+
+        if new_timeout:
+            self.ioloop.add_timeout(datetime.timedelta(seconds=10), self.report_state_with_timeout)
+
+    def report_state_with_timeout(self):
+        self.report_state(new_timeout=True)
+
+    def run(self, callback=None, interests=[]):
+        log.info("[%s] starting", self.identity)
+
+        # report READY state with interests
+        self.interests = interests
+        self.notify_callback = callback
+
+        self.stream.on_recv(self._cb_on_recv_multipart)
+        self.ioloop.add_timeout(datetime.timedelta(seconds=10), self.report_state_with_timeout)
+        self.ioloop.start()
+
+    def _cb_on_recv_multipart(self, message):
+        """
+            Receive a message on the Collector Router.
+        """
+        log.debug("Collector Router Message: %r" % (message), level=8)
+        cmd = message[0]
+
+        if cmd == b"STATE":
+            self.report_state()
+        elif self.state == b'BUSY':
+            self.report_state()
+        else:
+            job_uuid = message[1]
+            notification = message[2]
+
+            if not self.notify_callback == None:
+                self.notify_callback(cmd, job_uuid, notification)
+
+    def callback_done(self, job_uuid, result, threads = 0):
+        log.debug("Handler callback done for job %s: %r" % (job_uuid, result), level=8)
+        log.debug("Threads available: %d" % (threads), level=8)
+
+        if threads > 0:
+            self.state = b'READY'
+        else:
+            self.state = b'BUSY'
+
+        self.collector.send_multipart([b"DONE", job_uuid, result])
+        log.info("Job %s DONE by %s" % (job_uuid, self.identity))
+
+    def terminate(self):
+        log.info("[%s] shutting down", self.identity)
+        ioloop.IOLoop.instance().stop()
+        self.collector.close()

bonnie-0.3.6.tar.gz/bonnie/conf.py Added

@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging
+import os
+from optparse import OptionParser
+from ConfigParser import SafeConfigParser
+
+import bonnie
+log = bonnie.getLogger('bonnie.conf')
+from bonnie.translate import _
+
+class Conf(object):
+    cfg_parser = None
+
+    def __init__(self):
+        self.create_options()
+
+        self.config = SafeConfigParser()
+        if os.path.exists('/etc/bonnie/bonnie.conf'):
+            self.config.read('/etc/bonnie/bonnie.conf')
+        elif os.path.exists(os.path.abspath(os.path.dirname(__file__) + '/../conf/bonnie.conf')):
+            self.config.read(os.path.abspath(os.path.dirname(__file__) + '/../conf/bonnie.conf'))
+
+        self.defaults = Defaults()
+
+    def add_cli_parser_option_group(self, name):
+        return self.cli_parser.add_option_group(name)
+
+    def check_config(self, val=None):
+        """
+            Checks self.config_file or the filename passed using 'val'
+            and returns a SafeConfigParser instance if everything is OK.
+        """
+
+        if not val == None:
+            config_file = val
+        else:
+            config_file = self.config_file
+
+        if not os.access(config_file, os.R_OK):
+            log.error(_("Configuration file %s not readable") % config_file)
+
+        config = SafeConfigParser()
+        log.debug(_("Reading configuration file %s") % config_file, level=8)
+        try:
+            config.read(config_file)
+        except:
+            log.error(_("Invalid configuration file %s") % config_file)
+
+        if not config.has_section("bonnie"):
+            log.warning(_("No master configuration section [bonnie] in configuration file %s") % config_file)
+
+        return config
+
+    def create_options(self):
+        """
+            Create the OptionParser for the options passed to us from runtime
+            Command Line Interface.
+        """
+
+        # Enterprise Linux 5 does not have an "epilog" parameter to OptionParser
+        try:
+            self.cli_parser = OptionParser(epilog=epilog)
+        except:
+            self.cli_parser = OptionParser()
+
+        ##
+        ## Runtime Options
+        ##
+        runtime_group = self.cli_parser.add_option_group(_("Runtime Options"))
+        runtime_group.add_option(   "-c", "--config",
+                                    dest    = "config_file",
+                                    action  = "store",
+                                    default = "/etc/bonnie/bonnie.conf",
+                                    help    = _("Configuration file to use"))
+
+        runtime_group.add_option(   "-d", "--debug",
+                                    dest    = "debuglevel",
+                                    type    = 'int',
+                                    default = 0,
+                                    help    = _("Set the debugging " + \
+                                        "verbosity. Maximum is 9, tracing " + \
+                                        "protocols like LDAP, SQL and IMAP."))
+
+        runtime_group.add_option(   "-l",
+                                    dest    = "loglevel",
+                                    type    = 'str',
+                                    default = "CRITICAL",
+                                    help    = _("Set the logging level. " + \
+                                        "One of info, warn, error, " + \
+                                        "critical or debug"))
+
+        runtime_group.add_option(   "--logfile",
+                                    dest    = "logfile",
+                                    action  = "store",
+                                    default = "/var/log/bonnie/worker.log",
+                                    help    = _("Log file to use"))
+
+        runtime_group.add_option(   "-q", "--quiet",
+                                    dest    = "quiet",
+                                    action  = "store_true",
+                                    default = False,
+                                    help    = _("Be quiet."))
+
+    def finalize_conf(self,fatal=True):
+        self.parse_options(fatal=fatal)
+
+        # The defaults can some from;
+        # - a file we ship with the packages
+        # - a customly supplied file (by customer)
+        # - a file we write out
+        # - this python class
+
+        # This is where we check our parser for the defaults being set there.
+        self.set_defaults_from_cli_options()
+
+        self.options_set_from_config()
+
+        # Also set the cli options
+        if hasattr(self,'cli_keywords') and not self.cli_keywords == None:
+            for option in self.cli_keywords.__dict__.keys():
+                retval = False
+                if hasattr(self, "check_setting_%s" % (option)):
+                    exec("retval = self.check_setting_%s(%r)" % (option, self.cli_keywords.__dict__[option]))
+
+                    # The warning, error or confirmation dialog is in the check_setting_%s() function
+                    if not retval:
+                        continue
+
+                    log.debug(_("Setting %s to %r (from CLI, verified)") % (option, self.cli_keywords.__dict__[option]), level=8)
+                    setattr(self,option,self.cli_keywords.__dict__[option])
+                else:
+                    log.debug(_("Setting %s to %r (from CLI, not checked)") % (option, self.cli_keywords.__dict__[option]), level=8)
+                    setattr(self,option,self.cli_keywords.__dict__[option])
+
+    def get(self, section, key, default=None, quiet=False):
+        """
+            Get a configuration option from our store, the configuration file,
+            or an external source if we have some sort of function for it.
+
+            TODO: Include getting the value from plugins through a hook.
+        """
+        retval = False
+
+        if not self.cfg_parser:
+            self.read_config()
+
+        #log.debug(_("Obtaining value for section %r, key %r") % (section, key), level=8)
+
+        if self.cfg_parser.has_option(section, key):
+            try:
+                return self.cfg_parser.get(section, key)
+            except:
+                self.read_config()
+                return self.cfg_parser.get(section, key)
+
+        if not quiet:
+            log.warning(_("Option %s/%s does not exist in config file %s, pulling from defaults") % (section, key, self.config_file))
+            if hasattr(self.defaults, "%s_%s" % (section,key)):
+                return getattr(self.defaults, "%s_%s" % (section,key))
+            elif hasattr(self.defaults, "%s" % (section)):
+                if key in getattr(self.defaults, "%s" % (section)):
+                    _dict = getattr(self.defaults, "%s" % (section))
+                    return _dict[key]
+                else:
+                    log.warning(_("Option does not exist in defaults."))
+            else:
+                log.warning(_("Option does not exist in defaults."))
+
+        return default
+
+    def load_config(self, config):
+        """
+            Given a SafeConfigParser instance, loads a configuration
+            file and checks, then sets everything it can find.
+        """
+        for section in self.defaults.__dict__.keys():
+            if not config.has_section(section):
+                continue
+
+            for key in self.defaults.__dict__[section].keys():

bonnie-0.3.6.tar.gz/bonnie/daemon.py Added

@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import sys
+import grp
+import pwd
+import signal
+import traceback
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie')
+
+class BonnieDaemon(object):
+    """
+        A standard daemon process abstraction layer for Bonnie.
+
+        This class provides the following capabilities for Bonnie
+        daemons (through inheritance):
+
+        *   standard command-line options
+        *   :func:`dropping privileges <drop_privileges>`
+        *   :func:`signal handling <signal_handlers>`
+    """
+    pidfile = "/var/run/bonnie/bonnie.pid"
+
+    def __init__(self, *args, **kw):
+        daemon_group = conf.add_cli_parser_option_group("Daemon Options")
+
+        daemon_group.add_option(
+                "--fork",
+                dest    = "fork_mode",
+                action  = "store_true",
+                default = False,
+                help    = "Fork to the background."
+            )
+
+        daemon_group.add_option(
+                "-p",
+                "--pid-file",
+                dest    = "pidfile",
+                action  = "store",
+                default = self.pidfile,
+                help    = "Path to the PID file to use."
+            )
+
+        daemon_group.add_option(
+                "-u",
+                "--user",
+                dest    = "process_username",
+                action  = "store",
+                default = "kolab",
+                help    = "Run as user USERNAME",
+                metavar = "USERNAME"
+            )
+
+        daemon_group.add_option(
+                "-g",
+                "--group",
+                dest    = "process_groupname",
+                action  = "store",
+                default = "kolab",
+                help    = "Run as group GROUPNAME",
+                metavar = "GROUPNAME"
+            )
+
+        conf.finalize_conf()
+
+    def run(self, *args, **kw):
+        """
+            The daemon main loop. Override this function in a
+            :class:`BonnieDaemon` sub-class.
+        """
+        pass
+
+    def start(self, *args, **kw):
+        """
+            Start the daemon
+        """
+        exitcode = 0
+        terminate = True
+
+        if conf.fork_mode:
+            self.drop_privileges()
+
+        try:
+            pid = 1
+            if conf.fork_mode:
+                pid = daemonize()
+
+            if pid == 0:
+                self.write_pid()
+                self.signal_handlers()
+                self.run(*args, **kw)
+            elif not conf.fork_mode:
+                self.signal_handlers()
+                self.run(*args, **kw)
+            else:
+                terminate = False
+
+        except SystemExit, errcode:
+            terminate = False
+            exitcode = errcode
+
+        except KeyboardInterrupt:
+            exitcode = 1
+            log.info("Interrupted by user")
+
+        except (AttributeError, TypeError) as errmsg:
+            exitcode = 1
+            traceback.print_exc()
+            print >> sys.stderr, "Traceback occurred, please report a " + \
+                "bug at https://issues.kolab.org"
+
+        except:
+            exitcode = 2
+            traceback.print_exc()
+            print >> sys.stderr, "Traceback occurred, please report a " + \
+                "bug at https://issues.kolab.org"
+
+        if terminate:
+            self.terminate()
+
+        sys.exit(exitcode)
+
+    def terminate(self, *args, **kw):
+        """
+            Daemon shutdown function
+        """
+        self.remove_pid()
+
+    def signal_handlers(self):
+        """
+            Register process signal handlers
+        """
+        signal.signal(signal.SIGTERM, self.terminate)
+
+    def write_pid(self):
+        """
+            Write the process ID to the configured pid file
+        """
+        pid = os.getpid()
+        fp = open(conf.pidfile, 'w')
+        fp.write("%d\n" % (pid))
+        fp.close()
+
+    def remove_pid(self, *args, **kw):
+        """
+            Remove our PID file.
+        """
+        if os.access(conf.pidfile, os.R_OK):
+            try:
+                os.remove(conf.pidfile)
+            except:
+                pass
+
+        raise SystemExit
+
+    def drop_privileges(self):
+        try:
+            try:
+                (ruid, euid, suid) = os.getresuid()
+                (rgid, egid, sgid) = os.getresgid()
+            except AttributeError, errmsg:
+                ruid = os.getuid()
+                rgid = os.getgid()
+
+            if ruid == 0:
+                # Means we can setreuid() / setregid() / setgroups()
+                if rgid == 0:
+                    # Get group entry details
+                    try:
+                        (
+                            group_name,
+                            group_password,
+                            group_gid,
+                            group_members
+                        ) = grp.getgrnam(conf.process_groupname)
+
+                    except KeyError:
+                        print >> sys.stderr, "Group %s does not exist" % (conf.process_groupname)

bonnie-0.3.6.tar.gz/bonnie/dealer Added

bonnie-0.3.6.tar.gz/bonnie/dealer/__init__.py Added

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+import outputs
+
+import bonnie
+
+# pylint: disable=invalid-name
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.dealer')
+
+log.logfile = '/var/log/bonnie/dealer.log'
+
+
+class BonnieDealer(object):
+    def __init__(self, *args, **kw):
+        self.output_modules = {}
+
+        for _class in outputs.list_classes():
+            __class = _class()
+            self.output_modules[__class] = __class.register(
+                callback=self.register_output
+            )
+
+    def register_output(self, interests):
+        pass
+
+    def accept_notification(self, notification):
+        parsed = json.loads(notification)
+        event = parsed['event']
+        user = parsed['user'] if 'user' in parsed else None
+
+        # ignore globally excluded events
+        exclude_events = conf.get('dealer', 'input_exclude_events', None)
+        if exclude_events is not None:
+            exclude_events = exclude_events.split(',')
+        else:
+            exclude_events = []
+
+        if event in exclude_events:
+            log.debug("Excluding notification for event %r" % (event), level=8)
+            return False
+
+        # If there is no user, we know to parse the event -- there's no
+        # blacklisting users without a user.
+        if user is None:
+            log.debug("Including notification without user", level=8)
+            return True
+
+        blacklist_events = conf.get('dealer', 'blacklist_events', None)
+        if blacklist_events is not None:
+            blacklist_events = blacklist_events.split(',')
+        else:
+            blacklist_events = []
+
+        blacklist_users = conf.get('dealer', 'blacklist_users', None)
+        if blacklist_users is not None:
+            blacklist_users = blacklist_users.split(',')
+        else:
+            blacklist_users = []
+
+        # ignore blacklisted events for blacklisted users
+        if event in blacklist_events and user in blacklist_users:
+            log.debug(
+                "Excluding blacklisted event %r for blacklisted user %r" % (
+                    event,
+                    user
+                ),
+                level=8
+            )
+
+            return False
+
+        return True
+
+    def run(self, notification):
+        if self.accept_notification(notification):
+            output_modules = conf.get('dealer', 'output_modules')
+            output_modules = [x.strip() for x in output_modules.split(',')]
+            for _output in self.output_modules.keys():
+                if _output.name() in output_modules:
+                    _output.run(notification)
+        else:
+            log.info("Ignoring notification %s", notification)

bonnie-0.3.6.tar.gz/bonnie/dealer/outputs Added

bonnie-0.3.6.tar.gz/bonnie/dealer/outputs/__init__.py Added

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from file_output import FileOutput
+from zmq_output import ZMQOutput
+
+__all__ = [
+        'FileOutput',
+        'ZMQOutput'
+    ]
+
+def list_classes():
+    return [
+            FileOutput,
+            ZMQOutput
+        ]

bonnie-0.3.6.tar.gz/bonnie/dealer/outputs/file_output.py Added

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import socket
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.dealer.FileOutput')
+
+class FileOutput(object):
+    def __init__(self, *args, **kw):
+        self.file_output = conf.get("dealer", "file_path")
+
+    def name(self):
+        return 'file_output'
+
+    def register(self, *args, **kw):
+        return self.run
+
+    def run(self, notification):
+        log.debug("Notification received: %r" % (notification), level=9)
+        if self.file_output == None:
+            return notification
+
+        fp = open(self.file_output, 'a')
+        fp.write("%r\n" % (notification))
+        fp.close()

bonnie-0.3.6.tar.gz/bonnie/dealer/outputs/zmq_output.py Added

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import socket
+import zmq
+from zmq.eventloop import ioloop, zmqstream
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.dealer.ZMQOutput')
+
+class ZMQOutput(object):
+    def __init__(self, *args, **kw):
+        self.context = zmq.Context()
+
+        ioloop.install()
+
+        zmq_broker_address = conf.get('dealer', 'zmq_broker_address')
+
+        if zmq_broker_address == None:
+            zmq_broker_address = "tcp://localhost:5570"
+
+        self.dealer = self.context.socket(zmq.DEALER)
+        self.dealer.identity = (u"Dealer-%s-%s" % (socket.getfqdn(), os.getpid())).encode('ascii')
+        self.dealer.connect(zmq_broker_address)
+
+        self.dealer_stream = zmqstream.ZMQStream(self.dealer)
+        self.dealer_stream.on_recv(self.stop)
+
+    def name(self):
+        return 'zmq_output'
+
+    def register(self, *args, **kw):
+        return self.run
+
+    def run(self, notification):
+        log.debug("[%s] Notification received: %r" % (self.dealer.identity, notification), level=9)
+        self.dealer.send(notification)
+
+        ioloop.IOLoop.instance().start()
+
+    def stop(self, message, *args, **kw):
+        cmd = message[0]
+
+        if not cmd == b'ACK':
+            log.error("Unknown cmd %s" % (cmd))
+
+        ioloop.IOLoop.instance().stop()
+
+        self.dealer.close()
+        self.context.term()
+

bonnie-0.3.6.tar.gz/bonnie/logger.py Added

@@ -0,0 +1,186 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import grp
+import logging
+import logging.handlers
+import os
+import pwd
+import sys
+import time
+
+from bonnie.translate import _
+
+
+class Logger(logging.Logger):
+    """
+        The Bonnie version of a logger.
+
+        This class wraps the Python native logging library, adding to the
+        loglevel capabilities, a debuglevel capability.
+    """
+    debuglevel = 0
+    fork = False
+    logfile = '/var/log/bonnie/bonnie.log'
+    loglevel = logging.CRITICAL
+
+    if hasattr(sys, 'argv'):
+        for arg in sys.argv:
+            value = None
+            if '=' in arg:
+                (arg, value) = arg.split('=')[0:2]
+
+            if '-d' == arg or '--debug' == arg:
+                debuglevel = value if value is not None else -1
+                continue
+
+            if debuglevel == -1:
+                try:
+                    debuglevel = int(arg)
+                except ValueError:
+                    debuglevel = 0
+                    continue
+
+                loglevel = logging.DEBUG
+                continue
+
+            if '-l' == arg:
+                loglevel = -1
+                continue
+
+            if loglevel == -1:
+                if hasattr(logging, arg.upper()):
+                    loglevel = getattr(logging, arg.upper())
+                else:
+                    loglevel = logging.DEBUG
+
+            if '--logfile' == arg:
+                logfile = value
+                continue
+
+            if logfile is None:
+                logfile = arg
+                continue
+
+            if '--fork' == arg:
+                fork = True
+                continue
+
+    def __init__(self, *args, **kw):
+        if 'name' in kw:
+            name = kw['name']
+        elif len(args) == 1:
+            name = args[0]
+        else:
+            name = 'bonnie'
+
+        logging.Logger.__init__(self, name)
+
+        plaintextformatter = logging.Formatter(
+            "%(asctime)s %(name)s %(levelname)s %(message)s"
+        )
+
+        if not self.fork:
+            self.console_stdout = logging.StreamHandler(sys.stdout)
+            self.console_stdout.setFormatter(plaintextformatter)
+
+            self.addHandler(self.console_stdout)
+
+        if 'logfile' in kw:
+            self.logfile = kw['logfile']
+        elif self.logfile is None:
+            self.logfile = '/var/log/bonnie/bonnie.log'
+
+        self.setLevel(self.loglevel)
+
+        # Make sure (read: attempt to change) the permissions
+        try:
+            (ruid, euid, suid) = os.getresuid()
+            (rgid, egid, sgid) = os.getresgid()
+        except AttributeError:
+            ruid = os.getuid()
+            rgid = os.getgid()
+
+        if ruid == 0 or rgid == 0:
+            if os.path.isfile(self.logfile):
+                try:
+                    os.chown(
+                            self.logfile,
+                            pwd.getpwnam('cyrus')[2],
+                            grp.getgrnam('mail')[2]
+                        )
+                    os.chmod(self.logfile, 660)
+                except Exception:
+                    pass
+
+        # Make sure the log file exists
+        try:
+            fhandle = file(self.logfile, 'a')
+            try:
+                os.utime(self.logfile, None)
+            finally:
+                fhandle.close()
+
+            try:
+                filelog_handler = logging.FileHandler(filename=self.logfile)
+                filelog_handler.setFormatter(plaintextformatter)
+            except IOError, errmsg:
+                print >> sys.stderr, _(
+                    "Cannot log to file %s: %s"
+                ) % (
+                    self.logfile,
+                    errmsg
+                )
+
+            if not len(self.handlers) > 1:
+                try:
+                    self.addHandler(filelog_handler)
+                except Exception, errmsg:
+                    pass
+
+        except IOError, errmsg:
+            pass
+
+    def remove_stdout_handler(self):
+        if not self.fork:
+            self.console_stdout.close()
+            self.removeHandler(self.console_stdout)
+
+    def info(self, msg, *args):
+        # Suppress info messages from other applications according to debug
+        # level
+        if self.name.startswith('sqlalchemy') and self.debuglevel < 9:
+            return
+        if not self.name.startswith('bonnie') and self.debuglevel < 8:
+            return
+
+        self.log(logging.INFO, '[%d]: %s' % (os.getpid(), msg) % args)
+
+    def debug(self, msg, level=1, *args):
+        # Work around other applications not using various levels of debugging
+        if not self.name.startswith('bonnie') and not self.debuglevel == 9:
+            return
+
+        if level <= self.debuglevel:
+            # TODO: Not the way it's supposed to work!
+            self.log(logging.DEBUG, '[%d]: %s' % (os.getpid(), msg))
+
+
+logging.setLoggerClass(Logger)

bonnie-0.3.6.tar.gz/bonnie/plugins Added

bonnie-0.3.6.tar.gz/bonnie/plugins/__init__.py Added

@@ -0,0 +1,261 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import logging
+import os
+import pdb
+import sys
+import traceback
+
+import bonnie
+
+from bonnie.translate import _
+
+log = bonnie.getLogger('bonnie.plugins')
+conf = bonnie.getConf()
+
+class KolabPlugins(object):
+    """
+        Detects, loads and interfaces with plugins for different
+        Kolab components.
+    """
+    def __init__(self):
+        """
+            Searches the plugin directory for plugins, and loads
+            them into a list.
+        """
+        self.plugins = {}
+
+        for plugin_path in [
+                os.path.dirname(__file__),
+                '/usr/share/bonnie/plugins',
+                './bonnie/plugins'
+            ]:
+
+            if os.path.isdir(plugin_path):
+                for plugin in os.listdir(plugin_path):
+                    if os.path.isdir('%s/%s/' % (plugin_path,plugin,)):
+                        self.plugins[plugin] = False
+
+        self.check_plugins()
+
+    def check_plugins(self):
+        """
+            Checks all plugins in self.plugins and sets the values to
+            True (loadable) or False -- not enabled, not installed or
+            not loadable.
+        """
+        for plugin in self.plugins:
+            try:
+                exec("from bonnie.plugins import %s" % (plugin))
+                self.plugins[plugin] = True
+                self.load_plugins(plugins=[plugin])
+            except ImportError, e:
+                log.error(_("ImportError for plugin %s: %s") % (plugin,e))
+                traceback.print_exc()
+                self.plugins[plugin] = False
+            except RuntimeError, e:
+                log.error( _("RuntimeError for plugin %s: %s") % (plugin,e))
+                traceback.print_exc()
+                self.plugins[plugin] = False
+            except Exception, e:
+                log.error(_("Plugin %s failed to load (%s: %s)") % (plugin, e.__class__, e))
+                traceback.print_exc()
+            except:
+                traceback.print_exc()
+
+    def load_plugins(self, plugins=[]):
+        """
+            Loads plugins specified by a list of plugins or loads them all
+        """
+
+        if len(plugins) < 1:
+            plugins = self.plugins.keys()
+
+        for plugin in plugins:
+            if self.plugins[plugin]:
+                try:
+                    exec("self.%s = %s.Kolab%s()" % (plugin,plugin,plugin.capitalize()))
+                except:
+                    # TODO: A little better verbosity please!
+                    traceback.print_exc()
+
+    def set_defaults(self, defaults, plugins=[]):
+        """
+            Test for a function set_defaults() in all available and loaded plugins and execute plugin.set_defaults()
+        """
+        if len(plugins) < 1:
+            plugins = self.plugins.keys()
+
+        for plugin in plugins:
+            if not self.plugins[plugin]:
+                continue
+            if not hasattr(self,plugin):
+                continue
+
+            if hasattr(getattr(self,plugin),"set_defaults"):
+                try:
+                    getattr(self,plugin).set_defaults(defaults)
+                except TypeError, e:
+                    log.error(_("Cannot set defaults for plugin %s: %s") % (plugin,e))
+                except RuntimeError, e:
+                    log.error(_("Cannot set defaults for plugin %s: %s") % (plugin,e))
+                except:
+                    log.error(_("Cannot set defaults for plugin %s: Unknown Error") % (plugin))
+
+            else:
+                log.debug(_("Not setting defaults for plugin %s: No function 'set_defaults()'") % plugin, level=5)
+
+    def set_runtime(self, runtime, plugins=[]):
+        """
+            Set runtime variables from plugins, like 'i_did_all_this'
+        """
+        if len(plugins) < 1:
+            plugins = self.plugins.keys()
+
+        for plugin in plugins:
+            if not self.plugins[plugin]:
+                continue
+            if not hasattr(self,plugin):
+                continue
+
+            if hasattr(getattr(self,plugin),"set_runtime"):
+                try:
+                    getattr(self,plugin).set_runtime(runtime)
+                except RuntimeError, e:
+                    log.error(_("Cannot set runtime for plugin %s: %s") % (plugin,e))
+            else:
+                log.debug(_("Not setting runtime for plugin %s: No function 'set_runtime()'") % (plugin), level=5)
+
+    def add_options(self, parser, plugins=[]):
+        """
+            Add options specified in a plugin to parser. Takes a list of plugin names or does them all
+        """
+        if len(plugins) < 1:
+            plugins = self.plugins.keys()
+
+        for plugin in plugins:
+            if not self.plugins[plugin]:
+                continue
+            if not hasattr(self,plugin):
+                continue
+
+            if hasattr(getattr(self,plugin),"add_options"):
+                try:
+                    exec("self.%s.add_options(parser)" % plugin)
+                except RuntimeError, e:
+                    log.error(_("Cannot add options for plugin %s: %s") % (plugin,e))
+                except TypeError, e:
+                    log.error(_("Cannot add options for plugin %s: %s") % (plugin,e))
+            else:
+                    log.debug(_("Not adding options for plugin %s: No function 'add_options()'") % plugin, level=5)
+
+    def check_options(self, plugins=[]):
+        """
+            Executes plugin.check_plugins() for all enabled plugins or the list of plugin names specified.
+        """
+
+        if len(plugins) < 1:
+            plugins = self.plugins.keys()
+
+        for plugin in plugins:
+            if not self.plugins[plugin]:
+                continue
+            if not hasattr(self,plugin):
+                continue
+
+            if hasattr(getattr(self,plugin),"check_options"):
+                try:
+                    exec("self.%s.check_options()" % plugin)
+                except AttributeError, e:
+                    log.error(_("Cannot check options for plugin %s: %s") % (plugin,e))
+            else:
+                log.debug(_("Not checking options for plugin %s: No function 'check_options()'") % (plugin), level=5)
+
+    def plugin_check_setting(self, func, option, val, plugins=[]):
+        """
+            Checks one setting specified by 'option' against the 'val' it is passed by all plugins or by the list of plugins specified
+        """
+
+        if len(plugins) < 1:
+            plugins = self.plugins.keys()
+

bonnie-0.3.6.tar.gz/bonnie/translate.py Added

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+try:
+    from bonnie.constants import domain
+except ImportError, errmsg:
+    domain = 'bonnie'
+
+import gettext
+import os
+
+N_ = lambda x: x
+_ = lambda x: current.lgettext(x)
+
+localedir = '/usr/local/share/locale'
+current = gettext.translation(domain, localedir, fallback=True)
+
+def getDefaultLangs():
+    languages = []
+    for envar in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'):
+        val = os.environ.get(envar)
+        if val:
+            languages = val.split(':')
+            break
+    if 'C' not in languages:
+        languages.append('C')
+
+    # now normalize and expand the languages
+    nelangs = []
+    for lang in languages:
+        for nelang in gettext._expand_lang(lang):
+            if nelang not in nelangs:
+                nelangs.append(nelang)
+    return nelangs

bonnie-0.3.6.tar.gz/bonnie/utils.py Added

@@ -0,0 +1,244 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import urllib
+import urlparse
+import datetime
+import subprocess
+
+from dateutil.parser import parse as parse_date
+from dateutil.tz import tzutc
+
+from email import message_from_string
+from email.header import decode_header
+from email.utils import getaddresses
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.utils')
+
+
+def expand_uidset(uidset):
+    """
+        Expand the given UID set string into a complete set of values
+        Examples: 1,2,5 => [1,2,5] or 1:4 => [1,2,3,4]
+    """
+    _uids = []
+    for _uid in uidset.split(','):
+        if len(_uid.split(':')) > 1:
+            for __uid in range((int)(_uid.split(':')[0]), (int)(_uid.split(':')[1])+1):
+                _uids.append("%d" % (__uid))
+        else:
+            _uids.append(str(_uid))
+
+    return _uids
+
+
+def parse_imap_uri(uri):
+    """
+        Split the given URI string into its components
+    """
+    split_uri = urlparse.urlsplit(uri)
+
+    if len(split_uri.netloc.split('@')) == 3:
+        (username, domain, server) = split_uri.netloc.split('@')
+    elif len(split_uri.netloc.split('@')) == 2:
+        (username, server) = split_uri.netloc.split('@')
+        domain = None
+    elif len(split_uri.netloc.split('@')) == 1:
+        username = None
+        domain = None
+        server = split_uri.netloc
+
+    result = dict(user=username, domain=domain, host=server)
+
+    # First, .path == '/Calendar/Personal%20Calendar;UIDVALIDITY=$x[/;UID=$y]
+    # Take everything after the first slash, and omit any INBOX/ stuff.
+    path_str = '/'.join([x for x in split_uri.path.split('/') if not x == 'INBOX'][1:])
+    path_arr = path_str.split(';')
+    result['path'] = urllib.unquote(path_arr[0])
+
+    # parse the path/query parameters into a dict
+    param = dict()
+    for p in path_arr[1:]:
+        if '=' in p:
+            (key,val) = p.split('=', 2)
+            result[key] = urllib.unquote(val)
+
+    return result
+
+
+def decode_message_headers(message):
+    """
+        Copy headers from the given message object into a dict
+        structure and normalize the values into UTF-8 strings
+    """
+    headers = dict(message.items())
+
+    # split recipient headers into lists
+    for h in ['From','To','Cc','Bcc']:
+        if headers.has_key(h):
+            headers[h] = ['%s <%s>' % (name,address) for name,address in getaddresses(message.get_all(h))]
+
+    # decode header values into UTF-8
+    for k,value in headers.iteritems():
+        if isinstance(value, list):
+            headers[k] = [_decode_mime_header(x) for x in value]
+        else:
+            headers[k] = _decode_mime_header(value)
+
+    # replace content-type with the normalized value (good for searching)
+    headers['Content-Type'] = message.get_content_type()
+
+    # convert Date into UTC
+    if headers.has_key('Date'):
+        try:
+            date = parse_date(headers['Date']).astimezone(tzutc())
+            headers['Date'] = datetime.datetime.strftime(date, "%Y-%m-%dT%H:%M:%SZ")
+        except:
+            pass
+
+    return headers
+
+def _decode_mime_header(value):
+    """
+        Helper method to decode a single header string
+    """
+    return ' '.join(unicode(raw, charset) if charset is not None else raw for raw,charset in decode_header(value))
+
+
+def mail_message2dict(data):
+    """
+        Parse the given MIME message and return its contents as a dict
+    """
+    try:
+        message = message_from_string(data)
+    except Exception, e:
+        log.warning("Failed to parse MIME message: %r", e)
+        return dict(_data=data)
+
+    result = decode_message_headers(message)
+
+    if message.is_multipart():
+        for mime_part in message.walk():
+            # skip top level part
+            if not result.has_key('@parts'):
+                result['@parts'] = []
+                continue
+
+            part = dict(mime_part.items())
+            part['@body'] = mime_part.get_payload()
+
+            # convert payload into actual string data
+            if isinstance(part['@body'], list):
+                part['@body'] = (str(x) for x in part['@body'])
+
+            result['@parts'].append(part)
+    else:
+        result['@body'] = message.get_payload()
+
+    return result
+
+
+def imap_folder_path(uri):
+    """
+        Translate the folder name into a fully qualified folder path such as it
+        would be used by a cyrus administrator.
+    """
+    if isinstance(uri, str):
+        uri = parse_imap_uri(uri)
+
+    username = uri['user']
+    domain = uri['domain']
+    folder_name = uri['path']
+
+    # FIXME: prefix for shared folders; should be read from IMAP NAMESPACE command
+    shared_prefix = conf.get('imap', 'sharedprefix')
+
+    # Through filesystem
+    # To get the mailbox path, use:
+    # TODO: Assumption #1 is we are using virtual domains, and this domain does
+    # TODO: Assumption #2 is the mailbox in question is a user mailbox
+    # TODO: Assumption #3 is we use the unix hierarchy separator
+
+    # Translate the folder name in to a fully qualified folder path such as it
+    # would be used by a cyrus administrator.
+    #
+    # Other Users (covered, the netloc has the username the suffix is the
+    # original folder name).
+    if not username == None:
+        if folder_name == "INBOX":
+            folder_path = os.path.join('user', '%s@%s' % (username, domain))
+        else:
+            folder_path = os.path.join('user', username, '%s@%s' % (folder_name, domain))
+    # Shared Folders
+    else:
+        if folder_name.startswith(shared_prefix):
+            folder_path = folder_name[len(shared_prefix):]
+        else:
+            folder_path = folder_name
+
+    return folder_path

bonnie-0.3.6.tar.gz/bonnie/worker Added

bonnie-0.3.6.tar.gz/bonnie/worker/__init__.py Added

@@ -0,0 +1,391 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+import time
+import handlers
+import inputs
+import outputs
+import storage
+import signal
+
+from bonnie.translate import _
+from bonnie.daemon import BonnieDaemon
+from multiprocessing import Process
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.worker')
+
+class BonnieWorker(BonnieDaemon):
+    """
+        Bonnie Worker specific version of a :class:`Bonnie Daemon <bonnie.daemon.BonnieDaemon>`
+    """
+
+    #: The process ID file to use.
+    pidfile = "/var/run/bonnie/worker.pid"
+
+    def __init__(self, *args, **kw):
+        worker_group = conf.add_cli_parser_option_group("Worker Options")
+
+        worker_group.add_option(
+                "-n",
+                "--num-children",
+                dest    = "num_children",
+                action  = "store",
+                default = None,
+                help    = "Number of child processes to spawn"
+            )
+
+        super(BonnieWorker, self).__init__(*args, **kw)
+
+        self.children = []
+        self.manager = False
+        self.running = False
+
+    def run(self):
+        """
+            The Bonnie worker main loop, responsible for creating
+            :class:`worker processes
+            <bonnie.worker.BonnieWorkerProcess>`
+        """
+        num_children = conf.num_children or conf.get('worker', 'num_children')
+        if num_children is not None:
+            num_children = int(num_children)
+
+        if num_children is None or num_children < 1:
+            main = BonnieWorkerProcess()
+            self.children.append(main)
+            main.run()  # blocking
+        else:
+            conf.fork_mode = False
+            self.manager = True
+            self.running = True
+
+            while self.running:
+                # (re)start child worker processes
+                while len(self.children) < num_children:
+                    p = Process(target=self.run_child)
+                    self.children.append(p)
+                    p.start()
+
+                # check states of child processes
+                for p in self.children:
+                    if not p.is_alive():
+                        log.info("Restarting dead worker process %r", p.pid)
+                        self.children.remove(p)
+
+                time.sleep(10)
+
+        log.info("Shutting down worker manager")
+
+    def run_child(self):
+        """
+            Target method for :class:`worker processes
+            <bonnie.worker.BonnieWorkerProcess>`
+        """
+        BonnieWorkerProcess(as_child=True).run()
+
+    def terminate(self, *args, **kw):
+        """
+            Stop the worker daemon.
+        """
+        self.running = False
+        for p in self.children:
+            p.terminate()
+
+        if self.manager:
+            for p in self.children:
+                p.join()
+
+class BonnieWorkerProcess(object):
+    #: Holder of interests registered by handlers
+    handler_interests = { '_all': [] }
+
+    #: Holder of interests registered by input channel modules
+    input_interests = {}
+
+    #: Holder of interests registered by storage channel modules
+    storage_interests = {}
+
+    #: Holder of interests registered by output channel modules
+    output_interests = {}
+
+    #: Lists registered handler modules
+    handler_modules = {}
+    #: Lists registered input modules
+    input_modules = {}
+    #: Lists registered storage modules
+    storage_modules = {}
+    #: Lists registered output modules
+    output_modules = {}
+
+    output_exclude_events = []
+    def __init__(self, as_child=False, *args, **kw):
+        if as_child:
+            signal.signal(signal.SIGTERM, self.terminate)
+
+        for _class in handlers.list_classes():
+            __class = _class()
+            __class.register(callback=self.register_handler)
+            self.handler_modules[_class] = __class
+
+        for _class in inputs.list_classes():
+            __class = _class()
+            __class.register(callback=self.register_input)
+            self.input_modules[_class] = __class
+
+        output_modules = [x.strip() for x in conf.get('worker', 'output_modules', '').split(',')]
+
+        for _class in outputs.list_classes():
+            _output = _class()
+            if _output.name() in output_modules:
+                _output.register(callback=self.register_output)
+                self.output_modules[_class] = _output
+
+        storage_modules = [x.strip() for x in conf.get('worker', 'storage_modules', '').split(',')]
+        for _class in storage.list_classes():
+            _storage = _class()
+            if _storage.name() in storage_modules:
+                _storage.register(callback=self.register_storage)
+                self.storage_modules[_class] = _storage
+                self.storage = _storage
+
+        output_exclude_events = conf.get('worker', 'output_exclude_events', '')
+
+        self.output_exclude_events = [x.strip() for x in output_exclude_events.split(',')]
+
+    def event_notification(self, notification):
+        """
+            Input an event notification in to our process.
+
+            One by one, handlers that have shown an interest will be
+            handed over the entire event notification, and are expected
+            to return the (new) version of their event notification.
+        """
+
+        notification = json.loads(notification)
+
+        event = notification['event']
+
+        jobs = []
+
+        if self.handler_interests.has_key(event):
+            for interest in self.handler_interests[event]:
+                (notification, _jobs) = self.interest_callback(
+                        interest,
+                        notification
+                    )
+
+                if len(_jobs) > 0:
+                    log.debug(
+                            "Handler interest %r for event %s returns jobs: %r" % (

bonnie-0.3.6.tar.gz/bonnie/worker/constants.py Added

bonnie-0.3.6.tar.gz/bonnie/worker/handlers Added

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/__init__.py Added

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__package__ = 'bonnie.worker.handlers'
+
+import os
+
+import bonnie
+
+from base import HandlerBase
+from mailboxbase import MailboxHandlerBase
+from messagebase import MessageHandlerBase
+
+from aclchange import AclChangeHandler
+from flagsclear import FlagsClearHandler
+from flagsset import FlagsSetHandler
+from login import LoginHandler
+from logout import LogoutHandler
+from mailboxcreate import MailboxCreateHandler
+from mailboxdelete import MailboxDeleteHandler
+from mailboxrename import MailboxRenameHandler
+from mailboxsubscribe import MailboxSubscribeHandler
+from mailboxunsubscribe import MailboxUnsubscribeHandler
+from messageappend import MessageAppendHandler
+from messagecopy import MessageCopyHandler
+from messageexpire import MessageExpireHandler
+from messageexpunge import MessageExpungeHandler
+from messagemove import MessageMoveHandler
+from messagenew import MessageNewHandler
+from messageread import MessageReadHandler
+from messagetrash import MessageTrashHandler
+from quotaexceeded import QuotaExceededHandler
+from quotawithin import QuotaWithinHandler
+from quotachange import QuotaChangeHandler
+from changelog import ChangelogHandler
+
+__all__ = [
+        'AclChangeHandler',
+        'FlagsClearHandler',
+        'FlagsSetHandler',
+        'LoginHandler',
+        'LogoutHandler',
+        'MailboxHandlerBase',
+        'MailboxCreateHandler',
+        'MailboxDeleteHandler',
+        'MailboxRenameHandler',
+        'MailboxSubscribeHandler',
+        'MailboxUnsubscribeHandler',
+        'MessageAppendHandler',
+        'MessageCopyHandler',
+        'MessageExpireHandler',
+        'MessageExpungeHandler',
+        'MessageMoveHandler',
+        'MessageNewHandler',
+        'MessageReadHandler',
+        'MessageTrashHandler',
+        'QuotaExceededHandler',
+        'QuotaWithinHandler',
+        'QuotaChangeHandler'
+    ]
+
+conf = bonnie.getConf()
+
+def list_classes():
+    return [
+            AclChangeHandler,
+            FlagsClearHandler,
+            FlagsSetHandler,
+            LoginHandler,
+            LogoutHandler,
+            MailboxCreateHandler,
+            MailboxDeleteHandler,
+            MailboxRenameHandler,
+            MailboxSubscribeHandler,
+            MailboxUnsubscribeHandler,
+            MessageAppendHandler,
+            MessageCopyHandler,
+            MessageExpireHandler,
+            MessageExpungeHandler,
+            MessageMoveHandler,
+            MessageNewHandler,
+            MessageReadHandler,
+            MessageTrashHandler,
+            QuotaExceededHandler,
+            QuotaWithinHandler,
+            QuotaChangeHandler,
+            ChangelogHandler
+        ]

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/aclchange.py Added

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'AclChange'
+"""
+
+from bonnie.worker.handlers import MailboxHandlerBase
+
+class AclChangeHandler(MailboxHandlerBase):
+    """
+        Handler for an event that changes the ACL on an object.
+    """
+
+    event = 'AclChange'
+
+    def __init__(self, *args, **kw):
+        MailboxHandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/base.py Added

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import bonnie
+conf = bonnie.getConf()
+
+
+class HandlerBase(object):
+    features = []
+
+    def __init__(self, *args, **kw):
+        self.features = conf.get('bonnie', 'features')
+
+        if self.features is None:
+            self.features = ""
+
+        self.features = [x.strip() for x in self.features.split(',')]
+
+        log_name = 'bonnie.worker.handler'
+
+        if hasattr(self, 'event'):
+            log_name += '.%s' % (self.event)
+
+        self.log = bonnie.getLogger(log_name)
+
+    def register(self, callback):
+        interests = {
+                self.event: {
+                        'callback': self.run
+                    }
+            }
+
+        self.worker = callback(interests)
+
+    def run(self, notification):
+        # resolve user_id from storage?
+        resolve = False
+
+        if 'user' in notification:
+            if 'user_data' in notification:
+                user_data = notification['user_data']
+            else:
+                user_data = None
+
+            if 'user_id' not in notification:
+                resolve = True
+
+            elif notification['user_id'] is None:
+                resolve = True
+
+        if resolve:
+            notification['user_id'] = self.worker.storage.resolve_username(
+                notification['user'],
+                user_data,
+                force='user_data' in notification
+            )
+
+        # if storage has no entry, fetch user record from collector
+        if 'user' in notification:
+            if notification['user_id'] is None:
+                if 'user_data' not in notification:
+                    notification['user_data'] = None
+                    return (notification, [ b"GETUSERDATA" ])
+
+        # don't store user data in notification
+        if 'user_data' in notification:
+            del notification['user_data']
+
+        if 'user' in notification:
+            del notification['user']
+
+        return (notification, [])

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/changelog.py Added

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Changelog handler for groupware objects
+"""
+import re
+import time
+import bonnie
+
+from bonnie.worker.handlers import HandlerBase
+
+log = bonnie.getLogger('bonnie.worker.changelog')
+
+# timestamp (* 10) at the year 2014
+REVBASE = 13885344000
+
+
+class ChangelogHandler(HandlerBase):
+    events = ['MessageAppend', 'vnd.cmu.MessageMove']
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)
+
+    def register(self, callback):
+        kw = {'callback': self.run}
+        interests = dict((event, kw) for event in self.events)
+        self.worker = callback(interests)
+
+    def run(self, notification):
+        # message notifications require message headers
+        if not notification.has_key('messageHeaders'):
+            if notification['event'] == 'MessageAppend':
+                return (notification, [b"FETCH"])
+            else:
+                return (notification, [b"HEADER"])
+
+        # check if this is a groupware object
+        object_type = None
+
+        if notification.has_key('uidset'):
+            msguid = notification['uidset']
+        else:
+            msguid = None
+
+        # TODO: May need to iterate over message UIDs
+        if isinstance(msguid, list):
+            msguid = msguid.pop()
+
+        if notification['messageHeaders'].has_key(msguid):
+            headers = notification['messageHeaders'][msguid]
+        else:
+            headers = None
+
+        if headers:
+            if headers.has_key('X-Kolab-Type') and headers.has_key('Subject'):
+                match = re.match(
+                    r"application/x-vnd.kolab.(\w+)",
+                    headers['X-Kolab-Type']
+                )
+
+                if match:
+                    object_type = match.group(1)
+
+        # assign a revision number based on the current time
+        if object_type is not None:
+            notification['revision'] = int(round(time.time() * 10 - REVBASE))
+
+            # TODO: save object type and UUID in separate fields?
+            # These are translated into headers.X-Kolab-Type and
+            # headers.Subject by the output module
+
+        log.debug(
+            "Object type %r detected in event %r" % (
+                object_type,
+                notification['event']
+            ),
+            level=8
+        )
+
+        # TODO: suppress MessageTrash/MessageExpunge events when followed by a
+        # MessageAppend for the same object
+
+        return (notification, [])
+

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/flagsclear.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'FlagsClear'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class FlagsClearHandler(HandlerBase):
+    event = 'FlagsClear'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/flagsset.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'FlagsSet'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class FlagsSetHandler(HandlerBase):
+    event = 'FlagsSet'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/login.py Added

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'Login'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+
+class LoginHandler(HandlerBase):
+    """
+        A *Login* event notification handler.
+
+        The event notification is a JSON object with the following
+        layout::
+
+            {
+                    "event":"Login",
+                    "timestamp":"2014-11-27T10:45:38.201+01:00",
+                    "service":"imap",
+                    "serverDomain":"10.8.14.13",
+                    "serverPort":143,
+                    "clientIP":"10.8.13.10",
+                    "clientPort":45735,
+                    "uri":"imap://imapb13.example.org",
+                    "pid":14210,
+                    "user":"alexander.aachen@example.org",
+                    "vnd.cmu.sessionId":"cyrus-imapd-14210-1417081537-1-10541965771286434889"
+                }
+
+    """
+
+    event = 'Login'
+
+    def __init__(self, *args, **kw):
+        super(LoginHandler, self).__init__(*args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/logout.py Added

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'Logout'
+"""
+
+import datetime
+
+from dateutil.tz import tzutc
+from dateutil.parser import parse
+from bonnie.worker.handlers import HandlerBase
+
+
+class LogoutHandler(HandlerBase):
+    event = 'Logout'
+
+    def __init__(self, *args, **kw):
+        super(LogoutHandler, self).__init__(*args, **kw)
+
+    def run(self, notification):
+        # lookup corresponding Login event and update that record with
+        # logout_time and suppress separate logging of this event with
+        # notification['_suppress_output'] = True.
+        if 'vnd.cmu.sessionId' in notification:
+            now = datetime.datetime.now(tzutc())
+
+            results = self.worker.storage.select(
+                query=[
+                    ('event', '=', 'Login'),
+                    ('session_id', '=', notification['vnd.cmu.sessionId'])
+                ],
+                index='logstash-*',
+                doctype='logs',
+                fields='user,@timestamp',
+                limit=1
+            )
+
+            if results and results['total'] > 0:
+                login_event = results['hits'][0]
+
+                try:
+                    timestamp = parse(login_event['@timestamp'])
+                    self.log.debug(
+                        "Timestamp parsed to %r" % (timestamp),
+                        level=8
+                    )
+
+                except Exception, errmsg:
+                    self.log.error(
+                        "Could not parse timestamp %r: %r" % (
+                            login_event['@timestamp'],
+                            errmsg
+                        )
+                    )
+
+                    timestamp = now
+
+                delta = now - timestamp
+
+                duration = delta.days * 24 * 3600
+                duration += delta.seconds
+                duration += delta.microseconds / 1e6
+
+                # update Login event record
+                self.worker.storage.set(
+                    key=login_event['_id'],
+                    index=login_event['_index'],
+                    doctype=login_event['_doctype'],
+                    value={
+                        'logout_time': datetime.datetime.strftime(
+                            now,
+                            "%Y-%m-%dT%H:%M:%S.%fZ"
+                        ),
+                        'duration': duration
+                    }
+                )
+
+                notification['_suppress_output'] = True
+                return (notification, [])
+            else:
+                self.log.warning(
+                    "Could not find related Login event for session ID: %r" % (
+                        notification['vnd.cmu.sessionId']
+                    )
+                )
+
+            # push back into the job queue, the corresponding Login event may
+            # not yet have been processed.
+            return (notification, [b"POSTPONE"])
+
+        return super(LogoutHandler, self).run(notification)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/mailboxbase.py Added

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MailboxRename'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class MailboxHandlerBase(HandlerBase):
+    event = None
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)
+
+    def run(self, notification):
+        # call super for some basic notification processing
+        (notification, jobs) = super(MailboxHandlerBase, self).run(notification)
+
+        # mailbox notifications require metadata
+        if not notification.has_key('metadata'):
+            jobs.append(b"GETMETADATA")
+            return (notification, jobs)
+
+        # extract uniqueid from metadata -> triggers the storage module
+        if notification['metadata'].has_key('/shared/vendor/cmu/cyrus-imapd/uniqueid'):
+            notification['folder_uniqueid'] = notification['metadata']['/shared/vendor/cmu/cyrus-imapd/uniqueid']
+
+        return (notification, jobs)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/mailboxcreate.py Added

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MailboxCreate'
+"""
+
+import bonnie
+from bonnie.worker.handlers import MailboxHandlerBase
+
+class MailboxCreateHandler(MailboxHandlerBase):
+    event = 'MailboxCreate'
+
+    def __init__(self, *args, **kw):
+        MailboxHandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/mailboxdelete.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MailboxDelete'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class MailboxDeleteHandler(HandlerBase):
+    event = 'MailboxDelete'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/mailboxrename.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MailboxRename'
+"""
+
+from bonnie.worker.handlers import MailboxHandlerBase
+
+class MailboxRenameHandler(MailboxHandlerBase):
+    event = 'MailboxRename'
+
+    def __init__(self, *args, **kw):
+        MailboxHandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/mailboxsubscribe.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MailboxSubscribe'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class MailboxSubscribeHandler(HandlerBase):
+    event = 'MailboxSubscribe'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/mailboxunsubscribe.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MailboxUnsubscribe'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class MailboxUnsubscribeHandler(HandlerBase):
+    event = 'MailboxUnsubscribe'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/messageappend.py Added

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MessageAppend'
+"""
+
+from bonnie.worker.handlers import MessageHandlerBase
+
+
+class MessageAppendHandler(MessageHandlerBase):
+    event = 'MessageAppend'
+
+    def __init__(self, *args, **kw):
+        super(MessageAppendHandler, self).__init__(*args, **kw)
+
+    def run(self, notification):
+        # call super for some basic notification processing
+        (notification, jobs) = super(MessageAppendHandler, self).run(
+            notification
+        )
+
+        if 'messageContent' not in notification or \
+                notification['messageContent'] in [None, ""]:
+
+            self.log.debug("Adding FETCH job for " + self.event, level=8)
+            return (notification, [b"FETCH"])
+
+        return (notification, jobs)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/messagebase.py Added

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import bonnie
+
+from bonnie.utils import parse_imap_uri
+from bonnie.worker.handlers import HandlerBase
+
+conf = bonnie.getConf()
+
+class MessageHandlerBase(HandlerBase):
+    event = None
+
+    def __init__(self, *args, **kw):
+        super(MessageHandlerBase, self).__init__(*args, **kw)
+        self.log = bonnie.getLogger('bonnie.worker.' + self.event)
+
+    def run(self, notification):
+        # call super for some basic notification processing
+        (notification, jobs) = super(MessageHandlerBase, self).run(notification)
+
+        relevant = False
+
+        if 'archive' in self.features:
+            relevant = True
+
+        if 'backup' in self.features:
+            relevant = True
+
+        if not relevant:
+            return (notification, jobs)
+
+        # Insert the URI UID (if it exists) in to uidset for further handlers
+        if notification.has_key('uri') and notification.has_key('uidset'):
+            uri = parse_imap_uri(notification['uri'])
+
+            if uri.has_key('UID'):
+                notification['uidset'] = [ uri['UID'] ]
+
+        # message notifications require message headers
+        if not notification.has_key('messageHeaders'):
+            self.log.debug("Adding HEADER job for " + self.event, level=8)
+            jobs.append(b"HEADER")
+            return (notification, jobs)
+
+        return (notification, jobs)
+

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/messagecopy.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from bonnie.worker.handlers import MessageHandlerBase
+
+class MessageCopyHandler(MessageHandlerBase):
+    """
+        Handler for a *MessageCopy* event.
+    """
+
+    event = 'MessageCopy'
+
+    def __init__(self, *args, **kw):
+        MessageHandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/messageexpire.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MessageExpire'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class MessageExpireHandler(HandlerBase):
+    event = 'MessageExpire'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/messageexpunge.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MessageExpunge'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class MessageExpungeHandler(HandlerBase):
+    event = 'MessageExpunge'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/messagemove.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MessageMove'
+"""
+
+from bonnie.worker.handlers import MessageHandlerBase
+
+class MessageMoveHandler(MessageHandlerBase):
+    event = 'vnd.cmu.MessageMove'
+
+    def __init__(self, *args, **kw):
+        MessageHandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/messagenew.py Added

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MessageNew'
+"""
+
+import bonnie
+from bonnie.worker.handlers import MessageAppendHandler
+
+conf = bonnie.getConf()
+
+
+class MessageNewHandler(MessageAppendHandler):
+    event = 'MessageNew'
+
+    def __init__(self, *args, **kw):
+        super(MessageNewHandler, self).__init__(*args, **kw)
+
+    def run(self, notification):
+        # call super for some basic notification processing
+        (notification, jobs) = super(MessageAppendHandler, self).run(
+            notification
+        )
+
+        relevant = False
+
+        if 'archive' in self.features:
+            relevant = True
+
+        if 'backup' in self.features:
+            relevant = True
+
+        if not relevant:
+            return (notification, jobs)
+
+        if 'messageContent' not in notification or \
+                notification['messageContent'] in [None, ""]:
+            self.log.debug("Adding FETCH job for " + self.event, level=8)
+            return (notification, [b"FETCH"])
+
+        return (notification, jobs)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/messageread.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MessageRead'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class MessageReadHandler(HandlerBase):
+    event = 'MessageRead'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/messagetrash.py Added

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'MessageTrash'
+"""
+
+import bonnie
+from bonnie.worker.handlers import MessageHandlerBase
+
+class MessageTrashHandler(MessageHandlerBase):
+    event = 'MessageTrash'
+
+    def __init__(self, *args, **kw):
+        MessageHandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/quotachange.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'QuotaChange'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class QuotaChangeHandler(HandlerBase):
+    event = 'QuotaChange'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/quotaexceeded.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'QuotaExceeded'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class QuotaExceededHandler(HandlerBase):
+    event = 'QuotaExceeded'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/handlers/quotawithin.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+    Base handler for an event notification of type 'QuotaWithin'
+"""
+
+from bonnie.worker.handlers import HandlerBase
+
+class QuotaWithinHandler(HandlerBase):
+    event = 'QuotaWithin'
+
+    def __init__(self, *args, **kw):
+        HandlerBase.__init__(self, *args, **kw)

bonnie-0.3.6.tar.gz/bonnie/worker/inputs Added

bonnie-0.3.6.tar.gz/bonnie/worker/inputs/__init__.py Added

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from zmq_input import ZMQInput
+
+__all__ = [
+        'ZMQInput'
+    ]
+
+def list_classes():
+    return [
+            ZMQInput
+        ]

bonnie-0.3.6.tar.gz/bonnie/worker/inputs/zmq_input.py Added

@@ -0,0 +1,194 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+
+    This is a worker node pulling jobs from a ZMQ broker.
+
+"""
+
+import os
+import random
+import socket
+import time
+import zmq
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.worker.ZMQInput')
+
+class ZMQInput(object):
+    running = False
+
+    def __init__(self, *args, **kw):
+        self.state = b"READY"
+        self.job_uuid = None
+        self.lastping = 0
+        self.report_timestamp = 0
+        self.busy_start = 0
+
+    def name(self):
+        return 'zmq_input'
+
+    def register(self, *args, **kw):
+        pass
+
+    def report_state(self):
+        if self.busy_start < (time.time() - 90):
+            self.set_state_ready()
+
+        log.debug("[%s] reporting state: %s" % (self.identity, self.state), level=8)
+        message = [b"STATE", self.state]
+
+        if not self.job_uuid == None and self.state == b'BUSY':
+            message.append(self.job_uuid)
+
+        self.controller.send_multipart(message)
+        self.report_timestamp = time.time()
+
+    def run(self, callback=None, report=None):
+        self.identity = u"Worker-%s-%d" % (socket.getfqdn(),os.getpid())
+        log.info("[%s] starting", self.identity)
+
+        self.context = zmq.Context()
+
+        zmq_controller_address = conf.get('worker', 'zmq_controller_address')
+
+        if zmq_controller_address == None:
+            zmq_controller_address = "tcp://localhost:5572"
+
+        self.controller = self.context.socket(zmq.DEALER)
+        self.controller.identity = (self.identity).encode('ascii')
+        self.controller.connect(zmq_controller_address)
+
+        zmq_worker_router_address = conf.get('worker', 'zmq_worker_router_address')
+
+        if zmq_worker_router_address == None:
+            zmq_worker_router_address = "tcp://localhost:5573"
+
+        self.worker = self.context.socket(zmq.DEALER)
+        self.worker.identity = (self.identity).encode('ascii')
+        self.worker.connect(zmq_worker_router_address)
+
+        self.poller = zmq.Poller()
+        self.poller.register(self.controller, zmq.POLLIN)
+        self.poller.register(self.worker, zmq.POLLIN)
+
+        self.running = True
+        self.lastping = time.time()
+        self.report_state()
+
+        poller_timeout = int(conf.get('worker', 'zmq_poller_timeout', 100))
+
+        while self.running:
+            try:
+                sockets = dict(self.poller.poll(poller_timeout))
+            except KeyboardInterrupt, e:
+                log.info("zmq.Poller KeyboardInterrupt")
+                break
+            except Exception, e:
+                log.error("zmq.Poller error: %r", e)
+                sockets = dict()
+
+            now = time.time()
+
+            if self.report_timestamp < (time.time() - 10):
+                self.report_state()
+
+            if self.controller in sockets:
+                if sockets[self.controller] == zmq.POLLIN:
+                    _message = self.controller.recv_multipart()
+                    log.debug("[%s] Controller message: %r" % (self.identity, _message), level=9)
+
+                    if _message[0] == b"STATE":
+                        self.report_state()
+
+                    if _message[0] == b"TAKE":
+                        if not self.state == b"READY":
+                            self.report_state()
+
+                        else:
+                            self.job_uuid = _message[1]
+                            self.take_job(self.job_uuid)
+
+            if self.worker in sockets:
+                if sockets[self.worker] == zmq.POLLIN:
+                    _message = self.worker.recv_multipart()
+                    log.debug(
+                        "[%s] Worker message: %r" % (self.identity, _message),
+                        level=9
+                    )
+
+                    if _message[0] == "JOB":
+                        self.job_uuid = _message[1]
+
+                        # TODO: Sanity checking
+                        # if _message[1] == self.job_uuid:
+                        # jobs = []
+                        if callback is not None:
+                            (notification, jobs) = callback(_message[2])
+                        else:
+                            jobs = []
+
+                        if len(jobs) == 0:
+                            self.controller.send_multipart(
+                                [b"DONE", self.job_uuid]
+                            )
+
+                        else:
+                            log.debug(
+                                "[%s] Has jobs: %r" % (self.identity, jobs),
+                                level=8
+                            )
+
+                            if b'POSTPONE' in jobs:
+                                self.controller.send_multipart(
+                                    [b'POSTPONE', self.job_uuid]
+                                )
+
+                            else:
+                                self.controller.send_multipart([b'COLLECT', b" ".join(jobs), self.job_uuid, notification])
+
+                        self.set_state_ready()
+
+            if report is not None and self.lastping < (now - random.randint(300,600)):
+                report()
+                self.lastping = now
+
+        log.info("[%s] shutting down", self.identity)
+        self.worker.close()
+
+    def set_state_busy(self, _job_id):
+        log.debug("[%s] Set state to BUSY" % (self.identity), level=9)
+        self.report_timestamp = time.time()
+        self.busy_start = time.time()
+        self.state = b"BUSY"
+        self.job_uuid = _job_id
+
+    def set_state_ready(self):
+        log.debug("[%s] Set state to READY" % (self.identity), level=9)
+        self.report_timestamp = time.time()
+        self.state = b"READY"
+        self.job_uuid = None
+
+    def take_job(self, _job_id):
+        log.debug("[%s] Accept job %s" % (self.identity, _job_id), level=9)
+        self.set_state_busy(_job_id)
+        self.worker.send_multipart([b"GET", _job_id])

bonnie-0.3.6.tar.gz/bonnie/worker/outputs Added

bonnie-0.3.6.tar.gz/bonnie/worker/outputs/__init__.py Added

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+__all__ = []
+
+try:
+    from elasticsearch_output import ElasticSearchOutput
+    __all__.append('ElasticSearchOutput')
+except ImportError, errmsg:
+    pass
+
+try:
+    from riak_output import RiakOutput
+    __all__.append('RiakOutput')
+except ImportError, errmsg:
+    pass
+
+def list_classes():
+    classes = []
+
+    if 'ElasticSearchOutput' in __all__:
+        classes.append(ElasticSearchOutput)
+
+    if 'RiakOutput' in __all__:
+        classes.append(RiakOutput)
+
+    return classes

bonnie-0.3.6.tar.gz/bonnie/worker/outputs/elasticsearch_output.py Added

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import datetime
+
+from dateutil.parser import parse
+from dateutil.tz import tzutc
+
+import elasticsearch
+import json
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.worker.ElasticSearchOutput')
+
+class ElasticSearchOutput(object):
+    def __init__(self, *args, **kw):
+        elasticsearch_output_address = conf.get(
+                'worker',
+                'elasticsearch_output_address'
+            )
+
+        if elasticsearch_output_address == None:
+            elasticsearch_output_address = 'localhost'
+
+        self.es = elasticsearch.Elasticsearch(
+                host=elasticsearch_output_address
+            )
+
+    def name(self):
+        return 'elasticsearch_output'
+
+    def register(self, callback):
+        self.worker = callback({'_all': { 'callback': self.run }})
+
+    def notification_to_log(self, notification):
+        """
+            Convert the given event notification record into a valid log entry
+        """
+        keymap = {
+                'timestamp':                None,
+                'clientIP':                 'client_ip',
+                'clientPort':               None,
+                'serverPort':               None,
+                'serverDomain':             'domain',
+                'aclRights':                'acl_rights',
+                'aclSubject':               'acl_subject',
+                'mailboxID':                'mailbox_id',
+                'messageSize':              'message_size',
+                'messageHeaders':           None,
+                'messageContent':           None,
+                'bodyStructure':            None,
+                'metadata':                 None,
+                'acl':                      None,
+                'flagNames':                'flag_names',
+                'diskUsed':                 'disk_used',
+                'vnd.cmu.oldUidset':        'olduidset',
+                'vnd.cmu.sessionId':        'session_id',
+                'vnd.cmu.midset':           'message_id',
+                'vnd.cmu.unseenMessages':   'unseen_messages',
+            }
+
+        log = { '@version': bonnie.API_VERSION }
+
+        for key,val in notification.iteritems():
+            newkey = keymap[key] if keymap.has_key(key) else key
+            if newkey is not None:
+                # convert NIL values into None which is more appropriate
+                if isinstance(val, list):
+                    val = [x for x in val if not x == "NIL"]
+                elif val == "NIL":
+                    val = None
+
+                log[newkey] = val
+
+        return log
+
+    def run(self, notification):
+        # The output should have UTC timestamps, but gets "2014-05-16T12:55:53.870+02:00"
+        try:
+            timestamp = parse(notification['timestamp']).astimezone(tzutc())
+        except Exception, errmsg:
+            import traceback
+            log.error("Exception %r" % (errmsg))
+            log.error("%s" % (traceback.format_exc()))
+            timestamp = datetime.datetime.now(tzutc())
+
+        notification['@timestamp'] = datetime.datetime.strftime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ")
+        index = 'logstash-%s' % (datetime.datetime.strftime(timestamp, "%Y.%m.%d"))
+        jobs = []
+
+        # for notifications concerning multiple messages, create entries for each message
+        if notification.has_key('messageHeaders') and isinstance(notification['messageHeaders'], dict) and len(notification['messageHeaders']) > 0:
+            for uid,headers in notification['messageHeaders'].iteritems():
+                notification['uidset']  = uid
+                notification['headers'] = headers
+                notification['message'] = None
+                if notification.has_key('messageContent') and notification['messageContent'].has_key(uid):
+                    notification['message'] = notification['messageContent'][uid]
+                    # no need for bodystructure if we have the real message content
+                    notification.pop('bodyStructure', None)
+
+                # remove vnd.cmu.envelope if we have headers
+                notification.pop('vnd.cmu.envelope', None)
+
+                try:
+                    self.es.index(
+                        index=index,
+                        doc_type='logs',
+                        body=self.notification_to_log(notification)
+                    )
+                except Exception, e:
+                    log.warning("ES index(create or update) exception: %r", e)
+                    jobs.append(b'POSTPONE')
+                    break
+
+        else:
+            try:
+                self.es.index(
+                    index=index,
+                    doc_type='logs',
+                    body=self.notification_to_log(notification)
+                )
+            except Exception, e:
+                log.warning("ES index(create or update) exception: %r", e)
+                jobs.append(b'POSTPONE')
+
+        return (notification, jobs)

bonnie-0.3.6.tar.gz/bonnie/worker/outputs/riak_output.py Added

@@ -0,0 +1,142 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import datetime
+
+from dateutil.parser import parse
+from dateutil.tz import tzutc
+
+import riak
+import json
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.worker.RiakOutput')
+
+class RiakOutput(object):
+    def __init__(self, *args, **kw):
+        riak_output_address = conf.get(
+                'worker',
+                'riak_output_address'
+            )
+
+        if riak_output_address == None:
+            riak_output_address = 'localhost'
+
+        self.riak = riak.Riak(
+                host=riak_output_address
+            )
+
+    def name(self):
+        return 'riak_output'
+
+    def register(self, callback):
+        self.worker = callback({'_all': { 'callback': self.run }})
+
+    def notification_to_log(self, notification):
+        """
+            Convert the given event notification record into a valid log entry
+        """
+        keymap = {
+                'timestamp':                None,
+                'clientIP':                 'client_ip',
+                'clientPort':               None,
+                'serverPort':               None,
+                'serverDomain':             'domain',
+                'aclRights':                'acl_rights',
+                'aclSubject':               'acl_subject',
+                'mailboxID':                'mailbox_id',
+                'messageSize':              'message_size',
+                'messageHeaders':           None,
+                'messageContent':           None,
+                'bodyStructure':            None,
+                'metadata':                 None,
+                'acl':                      None,
+                'flagNames':                'flag_names',
+                'diskUsed':                 'disk_used',
+                'vnd.cmu.oldUidset':        'olduidset',
+                'vnd.cmu.sessionId':        'session_id',
+                'vnd.cmu.midset':           'message_id',
+                'vnd.cmu.unseenMessages':   'unseen_messages',
+            }
+
+        log = { '@version': bonnie.API_VERSION }
+
+        for key,val in notification.iteritems():
+            newkey = keymap[key] if keymap.has_key(key) else key
+            if newkey is not None:
+                # convert NIL values into None which is more appropriate
+                if isinstance(val, list):
+                    val = [x for x in val if not x == "NIL"]
+                elif val == "NIL":
+                    val = None
+
+                log[newkey] = val
+
+        return log
+
+    def run(self, notification):
+        # The output should have UTC timestamps, but gets "2014-05-16T12:55:53.870+02:00"
+        try:
+            timestamp = parse(notification['timestamp']).astimezone(tzutc())
+        except:
+            timestamp = datetime.datetime.now(tzutc())
+
+        notification['@timestamp'] = datetime.datetime.strftime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ")
+        index = 'logstash-%s' % (datetime.datetime.strftime(timestamp, "%Y.%m.%d"))
+        jobs = []
+
+        # for notifications concerning multiple messages, create entries for each message
+        if notification.has_key('messageHeaders') and isinstance(notification['messageHeaders'], dict) and len(notification['messageHeaders']) > 0:
+            for uid,headers in notification['messageHeaders'].iteritems():
+                notification['uidset']  = uid
+                notification['headers'] = headers
+                notification['message'] = None
+                if notification.has_key('messageContent') and notification['messageContent'].has_key(uid):
+                    notification['message'] = notification['messageContent'][uid]
+                    # no need for bodystructure if we have the real message content
+                    notification.pop('bodyStructure', None)
+
+                # remove vnd.cmu.envelope if we have headers
+                notification.pop('vnd.cmu.envelope', None)
+
+                try:
+                    self.riak.create(
+                        index=index,
+                        doc_type='logs',
+                        body=self.notification_to_log(notification)
+                    )
+                except Exception, errmsg:
+                    log.warning("Riak create exception: %r", e)
+                    jobs.append(b'POSTPONE')
+                    break
+
+        else:
+            try:
+                self.riak.create(
+                    index=index,
+                    doc_type='logs',
+                    body=self.notification_to_log(notification)
+                )
+            except Exception, errmsg:
+                log.warning("Riak create exception: %r", e)
+                jobs.append(b'POSTPONE')
+
+        return (notification, jobs)

bonnie-0.3.6.tar.gz/bonnie/worker/storage Added

bonnie-0.3.6.tar.gz/bonnie/worker/storage/__init__.py Added

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from caching import CachedDict
+
+__all__ = [
+        'CachedDict'
+    ]
+
+try:
+    from elasticsearch_storage import ElasticSearchStorage
+    __all__.append('ElasticSearchStorage')
+except ImportError, errmsg:
+    pass
+
+try:
+    from riak_storage import RiakStorage
+    __all__.append('RiakStorage')
+except ImportError, errmsg:
+    pass
+
+def list_classes():
+    classes = []
+
+    if 'ElasticSearchStorage' in __all__:
+        classes.append(ElasticSearchStorage)
+
+    if 'RiakStorage' in __all__:
+        classes.append(RiakStorage)
+
+    return classes

bonnie-0.3.6.tar.gz/bonnie/worker/storage/caching.py Added

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+
+class CachedDict(object):
+    """
+        dict-like class which drops items after the given TTL
+    """
+    def __init__(self, ttl=60):
+        # TODO: use memcache for distributed memory-based caching
+        self.ttl = ttl
+        self.data = {}
+
+    def remove(self, key):
+        self.data.remove(key)
+
+    def pop(self, key, default=None):
+        item = self.data.pop(key)
+        return item[0] if item is not None else default
+
+    def update(self, other):
+        expire = int(time.time()) + self.ttl
+        self.data.update(dict((k, (v, expire)) for k, v in other.items()))
+
+    def keys(self):
+        now = int(time.time())
+        return [k for k, v in self.data.items() if v[1] > now]
+
+    def values(self):
+        now = int(time.time())
+        return [v[0] for v in self.data.values() if v[1] > now]
+
+    def items(self):
+        now = int(time.time())
+        return dict((k, v[0]) for k, v in self.data.items() if v[1] > now).items()
+
+    def iteritems(self):
+        return self.items().iteritems()
+
+    def has_key(self, key):
+        return self.data.has_key(key) and self.data[key][1] > int(time.time())
+
+    def expunge(self):
+        now = int(time.time())
+        self.data = dict((k, v) for k, v in self.data.items() if v[1] > now)
+
+    def __getitem__(self, key):
+        return self.data[key][0]
+
+    def __setitem__(self, key, value):
+        self.data[key] = (value, int(time.time()) + self.ttl)
+
+    def __contains__(self, key):
+        return self.has_key(key)
+
+    def __len__(self):
+        return len(self.keys())
+
+    def __iter__(self):
+        return self.items().__iter__()

bonnie-0.3.6.tar.gz/bonnie/worker/storage/elasticsearch_storage.py Added

@@ -0,0 +1,600 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import re
+import json
+import urllib
+import hashlib
+import random
+import time
+import datetime
+import elasticsearch
+
+from dateutil.tz import tzutc
+from bonnie.utils import parse_imap_uri
+from bonnie.worker.storage import CachedDict
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.worker.ElasticSearchStorage')
+
+
+class ElasticSearchStorage(object):
+    """
+        Storage node writing object data into Elasticsearch
+    """
+    default_index = 'objects'
+    default_doctype = 'object'
+    folders_index = 'objects'
+    folders_doctype = 'folder'
+    users_index = 'objects'
+    users_doctype = 'user'
+
+    def __init__(self, *args, **kw):
+        elasticsearch_output_address = conf.get(
+                'worker',
+                'elasticsearch_storage_address'
+            )
+
+        if elasticsearch_output_address is None:
+            elasticsearch_output_address = 'localhost'
+
+        self.es = elasticsearch.Elasticsearch(
+                host=elasticsearch_output_address
+            )
+
+        # use dicts with automatic expiration for caching user/folder
+        # lookups
+        self.user_id_cache = CachedDict(300)
+        self.folder_id_cache = CachedDict(120)
+
+    def name(self):
+        return 'elasticsearch_storage'
+
+    def register(self, callback, **kw):
+        if callback is not None:
+            self.worker = callback(
+                interests={
+                    'uidset': {
+                        'callback': self.resolve_folder_uri
+                    },
+                    'folder_uniqueid': {
+                        'callback': self.resolve_folder_uri
+                    },
+                    'mailboxID': {
+                        'callback': self.resolve_folder_uri,
+                        'kw': {'attrib': 'mailboxID'}
+                    }
+                }
+            )
+
+    def get(self, key, index=None, doctype=None, stored_fields=None, **kw):
+        """
+            Standard API for accessing key/value storage
+        """
+        _index = index or self.default_index
+        _doctype = doctype or self.default_doctype
+
+        try:
+            res = self.es.get(
+                index=_index,
+                doc_type=_doctype,
+                id=key,
+                _source_include=stored_fields or '*'
+            )
+
+            log.debug(
+                "ES get result for %s/%s/%s: %r" % (
+                    _index,
+                    _doctype,
+                    key,
+                    res
+                ),
+                level=8
+            )
+
+            if res['found']:
+                result = self._transform_result(res)
+            else:
+                result = None
+
+        except elasticsearch.exceptions.NotFoundError, errmsg:
+            log.debug(
+                    "ES entry not found for %s/%s/%s: %r" % (
+                            _index,
+                            _doctype,
+                            key,
+                            errmsg
+                        )
+                )
+
+            result = None
+
+        except Exception, errmsg:
+            log.warning("ES get exception: %r" % (errmsg))
+            result = None
+
+        return result
+
+    def set(self, key, value, index=None, doctype=None, **kw):
+        """
+            Standard API for writing to key/value storage
+        """
+        _index = index or self.default_index
+        _doctype = doctype or self.default_doctype
+
+        try:
+            existing = self.es.get(
+                index=_index,
+                doc_type=_doctype,
+                id=key,
+                stored_fields=None
+            )
+
+            log.debug(
+                "ES get result for %s/%s/%s: %r" % (
+                    _index,
+                    _doctype,
+                    key,
+                    existing
+                ),
+                level=8
+            )
+
+        except elasticsearch.exceptions.NotFoundError, errmsg:
+            existing = None
+
+        except Exception, errmsg:
+            log.warning("ES get exception: %r" % (errmsg))
+            existing = None
+
+        if existing is None:
+            try:
+                ret = self.es.index(
+                    index=_index,
+                    doc_type=_doctype,
+                    id=key,
+                    body=value
+                )
+
+                log.debug(
+                    "Created ES object for %s/%s/%s: %r" % (
+                        _index,
+                        _doctype,
+                        key,
+                        ret
+                    ),
+                    level=8
+                )
+
+            except Exception, errmsg:
+                log.warning(
+                    "ES index(create or update) exception: %r" % (errmsg)
+                )
+
+                ret = None
+
+        else:
+            try:
+                ret = self.es.update(
+                    index=_index,
+                    doc_type=_doctype,
+                    id=key,

bonnie-0.3.6.tar.gz/bonnie/worker/storage/riak_storage.py Added

@@ -0,0 +1,577 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import datetime
+import hashlib
+import json
+import re
+import urllib
+import random
+import riak
+import time
+
+from dateutil.tz import tzutc
+from bonnie.utils import parse_imap_uri
+from bonnie.worker.storage import CachedDict
+
+import bonnie
+conf = bonnie.getConf()
+log = bonnie.getLogger('bonnie.worker.RiakStorage')
+
+class RiakStorage(object):
+    """
+        Storage node writing object data into Riak
+    """
+    default_index = 'objects'
+    default_doctype = 'object'
+    folders_index = 'objects'
+    folders_doctype = 'folder'
+    users_index = 'objects'
+    users_doctype = 'user'
+
+    def __init__(self, *args, **kw):
+        riak_output_address = conf.get(
+                'worker',
+                'riak_storage_address'
+            )
+
+        if riak_output_address == None:
+            riak_output_address = 'localhost'
+
+        self.es = riak.Elasticsearch(
+                host=riak_output_address
+            )
+
+        # use dicts with automatic expiration for caching user/folder
+        # lookups
+        self.user_id_cache = CachedDict(300)
+        self.folder_id_cache = CachedDict(120)
+
+    def name(self):
+        return 'riak_storage'
+
+    def register(self, callback, **kw):
+        if callback is not None:
+            self.worker = callback(
+                    interests = {
+                            'uidset': {
+                                    'callback': self.resolve_folder_uri
+                                },
+                            'folder_uniqueid': {
+                                    'callback': self.resolve_folder_uri
+                                },
+                            'mailboxID': {
+                                    'callback': self.resolve_folder_uri,
+                                    'kw': { 'attrib': 'mailboxID' }
+                                }
+                        }
+                )
+
+    def get(self, key, index=None, doctype=None, fields=None, **kw):
+        """
+            Standard API for accessing key/value storage
+        """
+        _index = index or self.default_index
+        _doctype = doctype or self.default_doctype
+        try:
+            res = self.riak.get(
+                    index = _index,
+                    doc_type = _doctype,
+                    id = key,
+                    _source_include = fields or '*'
+                )
+
+            log.debug(
+                    "Riak get result for %s/%s/%s: %r" % (
+                            _index,
+                            _doctype,
+                            key,
+                            res
+                        ),
+                    level = 8
+                )
+
+            if res['found']:
+                result = self._transform_result(res)
+            else:
+                result = None
+
+        except riak.exceptions.NotFoundError, errmsg:
+            log.debug(
+                    "Riak entry not found for %s/%s/%s: %r" % (
+                            _index,
+                            _doctype,
+                            key,
+                            errmsg
+                        )
+                )
+
+            result = None
+
+        except Exception, errmsg:
+            log.warning("Riak get exception: %r" % (errmsg))
+            result = None
+
+        return result
+
+    def set(self, key, value, index=None, doctype=None, **kw):
+        """
+            Standard API for writing to key/value storage
+        """
+        _index = index or self.default_index
+        _doctype = doctype or self.default_doctype
+
+        try:
+            existing = self.riak.get(
+                    index = _index,
+                    doc_type = _doctype,
+                    id = key,
+                    fields = None
+                )
+
+            log.debug(
+                    "Riak get result for %s/%s/%s: %r" % (
+                            _index,
+                            _doctype,
+                            key,
+                            existing
+                        ),
+                    level = 8
+                )
+
+        except riak.exceptions.NotFoundError, errmsg:
+            existing = None
+
+        except Exception, errmsg:
+            log.warning("Riak get exception: %r" % (errmsg))
+            existing = None
+
+        if existing is None:
+            try:
+                ret = self.riak.create(
+                        index = _index,
+                        doc_type = _doctype,
+                        id = key,
+                        body = value,
+                        consistency = 'one',
+                        replication = 'async'
+                    )
+
+                log.debug(
+                        "Created ES object for %s/%s/%s: %r" % (
+                                _index,
+                                _doctype,
+                                key,
+                                ret
+                            ),
+                        level = 8
+                    )
+
+            except Exception, errmsg:
+                log.warning("Riak create exception: %r" % (errmsg))
+                ret = None
+
+        else:
+            try:
+                ret = self.riak.update(
+                        index = _index,
+                        doc_type = _doctype,
+                        id = key,
+                        body = { 'doc': value },
+                        consistency = 'one',
+                        replication = 'async'

bonnie-0.3.6.tar.gz/broker.py Added

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from bonnie.broker import BonnieBroker
+
+if __name__ == "__main__":
+    broker = BonnieBroker()
+    broker.start()

bonnie-0.3.6.tar.gz/collector.py Added

@@ -0,0 +1,26 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from bonnie.collector import BonnieCollector
+
+if __name__ == "__main__":
+    collector = BonnieCollector()
+    collector.start()

bonnie-0.3.6.tar.gz/conf Added

bonnie-0.3.6.tar.gz/conf/bonnie.conf Added

@@ -0,0 +1,84 @@
+[bonnie]
+; Which features do we enable?
+;
+; Valid options: 'archive', 'audit', 'backup', 'dlp'
+;
+; Supported options: None
+features = archive, audit, backup, dlp
+
+[broker]
+
+; The retention of jobs
+job_retention = 300
+
+; The broker's dealer router bind address
+zmq_dealer_router_bind_address = tcp://*:5570
+
+; The broker's collector router bind address
+zmq_collector_router_bind_address = tcp://*:5571
+
+; The broker's worker controller bind address
+zmq_worker_controller_router_bind_address = tcp://*:5572
+
+; The broker's worker router bind address
+zmq_worker_router_bind_address = tcp://*:5573
+
+zmq_poller_timeout = 100
+;state_sql_uri = sqlite://
+state_sql_uri = mysql://bonnie:Welcome2KolabSystems@127.0.0.1/bonnie
+
+[collector]
+; Size of the worker queue -- take at most this many jobs at any one time.
+num_threads = 5
+
+; The input
+input_modules = zmq_input
+
+; Where's the input's broker?
+zmq_broker_address = tcp://localhost:5571
+
+; Not currently used
+zmq_poller_timeout = 100
+
+[dealer]
+; How do we output our notifications?
+output_modules = zmq_output
+
+; And where to?
+zmq_broker_address = tcp://localhost:5570
+
+; Blacklist the following events for <blacklist_users> -- prevents infinite
+; loops.
+blacklist_events = Login,Logout,AclChange
+blacklist_users = cyrus-admin
+input_exclude_events =
+
+[worker]
+; The number of child processes to spawn.
+num_children = 0
+
+; How do we get our notifications to work on?
+input_modules = zmq_input
+
+; What do we use to store intermediate data?
+storage_modules = elasticsearch_storage
+
+; What do we use to put the result of what we do?
+output_modules = elasticsearch_output
+
+; Supply a comma or comma-space separated list of events we need not put out.
+output_exclude_events = MessageExpunge
+
+; Where's our controller?
+zmq_controller_address = tcp://localhost:5572
+
+; Where's our router?
+zmq_worker_router_address = tcp://localhost:5573
+
+zmq_poller_timeout = 100
+
+elasticsearch_output_address = localhost
+elasticsearch_storage_address = localhost
+
+[imap]
+sharedprefix = Shared Folders/

bonnie-0.3.6.tar.gz/contrib Added

bonnie-0.3.6.tar.gz/contrib/bonnie-broker.sysconfig Added

bonnie-0.3.6.tar.gz/contrib/bonnie-broker.systemd Added

bonnie-0.3.6.tar.gz/contrib/bonnie-broker.sysvinit Added

@@ -0,0 +1,121 @@
+#! /bin/bash
+#
+# bonnie-broker  Start/Stop the Bonnie Broker daemon
+#
+# chkconfig:        - 65 10
+# description:      The Bonnie Broker daemon is a message broker.
+# processname: bonnie-broker
+
+### BEGIN INIT INFO
+# Provides: bonnie-brokerd
+# Default-Start: -
+# Default-Stop: 0 1 2 6
+# Required-Start: $remote_fs $local_fs $network
+# Required-Stop: $remote_fs $local_fs $network
+# Short-Description: Start/Stop the Bonnie Broker daemon
+# Description:      The Bonnie Broker daemon is a message broker.
+### END INIT INFO
+
+# Source function library.
+if [ -f /etc/init.d/functions ]; then
+    . /etc/init.d/functions
+fi
+
+# Source our configuration file for these variables.
+FLAGS="--fork -l warning"
+USER="bonnie"
+GROUP="bonnie"
+
+if [ -f /etc/sysconfig/bonnie-broker ] ; then
+    . /etc/sysconfig/bonnie-broker
+fi
+
+if [ -f /etc/default/bonnie-broker ]; then
+    . /etc/default/bonnie-broker
+fi
+
+RETVAL=0
+
+# Set up some common variables before we launch into what might be
+# considered boilerplate by now.
+prog=bonnie-broker
+path=/usr/sbin/bonnie-broker
+lockfile=/var/lock/subsys/$prog
+pidfile=/var/run/bonnie/bonnie-broker.pid
+
+[ ! -d "$(dirname ${pidfile})" ] && mkdir -p $(dirname ${pidfile})
+chown ${USER}:${GROUP} $(dirname ${pidfile})
+
+start() {
+    [ -x $path ] || exit 5
+    echo -n $"Starting $prog: "
+    daemon $DAEMONOPTS $path -p $pidfile $FLAGS
+    RETVAL=$?
+    echo
+    [ $RETVAL -eq 0 ] && touch $lockfile
+    return $RETVAL
+}
+
+stop() {
+    echo -n $"Stopping $prog: "
+    killproc -p $pidfile $prog
+    RETVAL=$?
+    echo
+    [ $RETVAL -eq 0 ] && rm -f $lockfile
+    return $RETVAL
+}
+
+restart() {
+    stop
+    start
+}
+
+reload() {
+    restart
+}
+
+force_reload() {
+    restart
+}
+
+rh_status() {
+    # run checks to determine if the service is running or use generic status
+    status -p $pidfile $prog
+}
+
+rh_status_q() {
+    rh_status >/dev/null 2>&1
+}
+
+case "$1" in
+  start)
+    rh_status_q && exit 0
+    start
+    ;;
+  stop)
+    rh_status_q || exit 0
+    stop
+    ;;
+  restart)
+    restart
+    ;;
+  reload)
+    rh_status_q || exit 7
+    reload
+    ;;
+  force-reload)
+    force_reload
+    ;;
+  status)
+    rh_status
+    ;;
+  condrestart|try-restart)
+    rh_status_q || exit 0
+    restart
+    ;;
+  *)
+    echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+    exit 2
+esac
+
+exit $?

bonnie-0.3.6.tar.gz/contrib/bonnie-collector.sysconfig Added

bonnie-0.3.6.tar.gz/contrib/bonnie-collector.systemd Added

bonnie-0.3.6.tar.gz/contrib/bonnie-collector.sysvinit Added

@@ -0,0 +1,121 @@
+#! /bin/bash
+#
+# bonnie-collector  Start/Stop the Bonnie Broker daemon
+#
+# chkconfig:        - 65 10
+# description:      The Bonnie Broker daemon is a message collector.
+# processname: bonnie-collector
+
+### BEGIN INIT INFO
+# Provides: bonnie-collectord
+# Default-Start: -
+# Default-Stop: 0 1 2 6
+# Required-Start: $remote_fs $local_fs $network
+# Required-Stop: $remote_fs $local_fs $network
+# Short-Description: Start/Stop the Bonnie Broker daemon
+# Description:      The Bonnie Broker daemon is a message collector.
+### END INIT INFO
+
+# Source function library.
+if [ -f /etc/init.d/functions ]; then
+    . /etc/init.d/functions
+fi
+
+# Source our configuration file for these variables.
+FLAGS="--fork -l warning"
+USER="bonnie"
+GROUP="bonnie"
+
+if [ -f /etc/sysconfig/bonnie-collector ] ; then
+    . /etc/sysconfig/bonnie-collector
+fi
+
+if [ -f /etc/default/bonnie-collector ]; then
+    . /etc/default/bonnie-collector
+fi
+
+RETVAL=0
+
+# Set up some common variables before we launch into what might be
+# considered boilerplate by now.
+prog=bonnie-collector
+path=/usr/sbin/bonnie-collector
+lockfile=/var/lock/subsys/$prog
+pidfile=/var/run/bonnie/bonnie-collector.pid
+
+[ ! -d "$(dirname ${pidfile})" ] && mkdir -p $(dirname ${pidfile})
+chown ${USER}:${GROUP} $(dirname ${pidfile})
+
+start() {
+    [ -x $path ] || exit 5
+    echo -n $"Starting $prog: "
+    daemon $DAEMONOPTS $path -p $pidfile $FLAGS
+    RETVAL=$?
+    echo
+    [ $RETVAL -eq 0 ] && touch $lockfile
+    return $RETVAL
+}
+
+stop() {
+    echo -n $"Stopping $prog: "
+    killproc -p $pidfile $prog
+    RETVAL=$?
+    echo
+    [ $RETVAL -eq 0 ] && rm -f $lockfile
+    return $RETVAL
+}
+
+restart() {
+    stop
+    start
+}
+
+reload() {
+    restart
+}
+
+force_reload() {
+    restart
+}
+
+rh_status() {
+    # run checks to determine if the service is running or use generic status
+    status -p $pidfile $prog
+}
+
+rh_status_q() {
+    rh_status >/dev/null 2>&1
+}
+
+case "$1" in
+  start)
+    rh_status_q && exit 0
+    start
+    ;;
+  stop)
+    rh_status_q || exit 0
+    stop
+    ;;
+  restart)
+    restart
+    ;;
+  reload)
+    rh_status_q || exit 7
+    reload
+    ;;
+  force-reload)
+    force_reload
+    ;;
+  status)
+    rh_status
+    ;;
+  condrestart|try-restart)
+    rh_status_q || exit 0
+    restart
+    ;;
+  *)
+    echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+    exit 2
+esac
+
+exit $?

bonnie-0.3.6.tar.gz/contrib/bonnie-worker.sysconfig Added

bonnie-0.3.6.tar.gz/contrib/bonnie-worker.systemd Added

bonnie-0.3.6.tar.gz/contrib/bonnie-worker.sysvinit Added

@@ -0,0 +1,121 @@
+#! /bin/bash
+#
+# bonnie-worker  Start/Stop the Bonnie Broker daemon
+#
+# chkconfig:        - 65 10
+# description:      The Bonnie Broker daemon is a message worker.
+# processname: bonnie-worker
+
+### BEGIN INIT INFO
+# Provides: bonnie-workerd
+# Default-Start: -
+# Default-Stop: 0 1 2 6
+# Required-Start: $remote_fs $local_fs $network
+# Required-Stop: $remote_fs $local_fs $network
+# Short-Description: Start/Stop the Bonnie Broker daemon
+# Description:      The Bonnie Broker daemon is a message worker.
+### END INIT INFO
+
+# Source function library.
+if [ -f /etc/init.d/functions ]; then
+    . /etc/init.d/functions
+fi
+
+# Source our configuration file for these variables.
+FLAGS="--fork -l warning"
+USER="bonnie"
+GROUP="bonnie"
+
+if [ -f /etc/sysconfig/bonnie-worker ] ; then
+    . /etc/sysconfig/bonnie-worker
+fi
+
+if [ -f /etc/default/bonnie-worker ]; then
+    . /etc/default/bonnie-worker
+fi
+
+RETVAL=0
+
+# Set up some common variables before we launch into what might be
+# considered boilerplate by now.
+prog=bonnie-worker
+path=/usr/sbin/bonnie-worker
+lockfile=/var/lock/subsys/$prog
+pidfile=/var/run/bonnie/bonnie-worker.pid
+
+[ ! -d "$(dirname ${pidfile})" ] && mkdir -p $(dirname ${pidfile})
+chown ${USER}:${GROUP} $(dirname ${pidfile})
+
+start() {
+    [ -x $path ] || exit 5
+    echo -n $"Starting $prog: "
+    daemon $DAEMONOPTS $path -p $pidfile $FLAGS
+    RETVAL=$?
+    echo
+    [ $RETVAL -eq 0 ] && touch $lockfile
+    return $RETVAL
+}
+
+stop() {
+    echo -n $"Stopping $prog: "
+    killproc -p $pidfile $prog
+    RETVAL=$?
+    echo
+    [ $RETVAL -eq 0 ] && rm -f $lockfile
+    return $RETVAL
+}
+
+restart() {
+    stop
+    start
+}
+
+reload() {
+    restart
+}
+
+force_reload() {
+    restart
+}
+
+rh_status() {
+    # run checks to determine if the service is running or use generic status
+    status -p $pidfile $prog
+}
+
+rh_status_q() {
+    rh_status >/dev/null 2>&1
+}
+
+case "$1" in
+  start)
+    rh_status_q && exit 0
+    start
+    ;;
+  stop)
+    rh_status_q || exit 0
+    stop
+    ;;
+  restart)
+    restart
+    ;;
+  reload)
+    rh_status_q || exit 7
+    reload
+    ;;
+  force-reload)
+    force_reload
+    ;;
+  status)
+    rh_status
+    ;;
+  condrestart|try-restart)
+    rh_status_q || exit 0
+    restart
+    ;;
+  *)
+    echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
+    exit 2
+esac
+
+exit $?

bonnie-0.3.6.tar.gz/contrib/bonnie.tmpfiles.d.conf Added

bonnie-0.3.6.tar.gz/dealer-async.py Added

@@ -0,0 +1,35 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import sys
+
+from bonnie.dealer import BonnieDealer
+
+if __name__ == "__main__":
+    # pylint: disable=invalid-name
+    notification = sys.stdin.read().strip()
+
+    newpid = os.fork()
+
+    if newpid == 0:
+        dealer = BonnieDealer()
+        dealer.run(notification)

bonnie-0.3.6.tar.gz/dealer-sync.py Added

@@ -0,0 +1,38 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import sys
+
+from bonnie.dealer import BonnieDealer
+
+if __name__ == "__main__":
+    # pylint: disable=invalid-name
+    notification = sys.stdin.read().strip()
+
+    dealer = BonnieDealer()
+    dealer.run(notification)
+
+#    newpid = os.fork()
+#
+#    if newpid == 0:
+#        dealer = BonnieDealer()
+#        dealer.run(notification)

bonnie-0.3.6.tar.gz/docs Added

bonnie-0.3.6.tar.gz/docs/Makefile Added

@@ -0,0 +1,153 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	-rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Bonnie.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Bonnie.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/Bonnie"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Bonnie"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."

bonnie-0.3.6.tar.gz/docs/_static Added

bonnie-0.3.6.tar.gz/docs/_static/basic.css_t Added

@@ -0,0 +1,541 @@
+/*
+ * basic.css
+ * ~~~~~~~~~
+ *
+ * Sphinx stylesheet -- basic theme.
+ *
+ * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS.
+ * :license: BSD, see LICENSE for details.
+ *
+ */
+
+/* -- main layout ----------------------------------------------------------- */
+
+div.clearer {
+    clear: both;
+}
+
+/* -- relbar ---------------------------------------------------------------- */
+
+div.related {
+    width: 100%;
+    font-size: 90%;
+}
+
+div.related h3 {
+    display: none;
+}
+
+div.related ul {
+    margin: 0;
+    padding: 0 0 0 10px;
+    list-style: none;
+}
+
+div.related li {
+    display: inline;
+}
+
+div.related li.right {
+    float: right;
+    margin-right: 5px;
+}
+
+/* -- sidebar --------------------------------------------------------------- */
+
+div.sphinxsidebarwrapper {
+    padding: 10px 5px 0 10px;
+}
+
+div.sphinxsidebar {
+    float: left;
+    width: 230px;
+    margin-left: -100%;
+    font-size: 90%;
+}
+
+div.sphinxsidebar ul {
+    list-style: none;
+}
+
+div.sphinxsidebar ul ul,
+div.sphinxsidebar ul.want-points {
+    margin-left: 20px;
+    list-style: square;
+}
+
+div.sphinxsidebar ul ul {
+    margin-top: 0;
+    margin-bottom: 0;
+}
+
+div.sphinxsidebar form {
+    margin-top: 10px;
+}
+
+div.sphinxsidebar input {
+    border: 1px solid #98dbcc;
+    font-family: sans-serif;
+    font-size: 1em;
+}
+
+div.sphinxsidebar #searchbox input[type="text"] {
+    width: 170px;
+}
+
+div.sphinxsidebar #searchbox input[type="submit"] {
+    width: 30px;
+}
+
+img {
+    max-width: 100%;
+    border: 0;
+}
+
+/* -- search page ----------------------------------------------------------- */
+
+ul.search {
+    margin: 10px 0 0 20px;
+    padding: 0;
+}
+
+ul.search li {
+    padding: 5px 0 5px 20px;
+    background-image: url(file.png);
+    background-repeat: no-repeat;
+    background-position: 0 7px;
+}
+
+ul.search li a {
+    font-weight: bold;
+}
+
+ul.search li div.context {
+    color: #888;
+    margin: 2px 0 0 30px;
+    text-align: left;
+}
+
+ul.keywordmatches li.goodmatch a {
+    font-weight: bold;
+}
+
+/* -- index page ------------------------------------------------------------ */
+
+table.contentstable {
+    width: 90%;
+}
+
+table.contentstable p.biglink {
+    line-height: 150%;
+}
+
+a.biglink {
+    font-size: 1.3em;
+}
+
+span.linkdescr {
+    font-style: italic;
+    padding-top: 5px;
+    font-size: 90%;
+}
+
+/* -- general index --------------------------------------------------------- */
+
+table.indextable {
+    width: 100%;
+}
+
+table.indextable td {
+    text-align: left;
+    vertical-align: top;
+}
+
+table.indextable dl, table.indextable dd {
+    margin-top: 0;
+    margin-bottom: 0;
+}
+
+table.indextable tr.pcap {
+    height: 10px;
+}
+
+table.indextable tr.cap {
+    margin-top: 10px;
+    background-color: #f2f2f2;
+}
+
+img.toggler {
+    margin-right: 3px;
+    margin-top: 3px;
+    cursor: pointer;
+}
+
+div.modindex-jumpbox {
+    border-top: 1px solid #ddd;
+    border-bottom: 1px solid #ddd;
+    margin: 1em 0 1em 0;
+    padding: 0.4em;
+}
+
+div.genindex-jumpbox {
+    border-top: 1px solid #ddd;
+    border-bottom: 1px solid #ddd;
+    margin: 1em 0 1em 0;
+    padding: 0.4em;
+}
+
+/* -- general body styles --------------------------------------------------- */
+
+a.headerlink {
+    visibility: hidden;
+}
+
+h1:hover > a.headerlink,
+h2:hover > a.headerlink,
+h3:hover > a.headerlink,
+h4:hover > a.headerlink,
+h5:hover > a.headerlink,
+h6:hover > a.headerlink,

bonnie-0.3.6.tar.gz/docs/_static/images Added

bonnie-0.3.6.tar.gz/docs/_static/images/bonnie_jobs_done-pinpoint=1417359241,1417411037.png Added

bonnie-0.3.6.tar.gz/docs/_static/images/bonnie_jobs_pending-pinpoint=1417359574,1417411771.png Added

bonnie-0.3.6.tar.gz/docs/about Added

bonnie-0.3.6.tar.gz/docs/about/archival.rst Added

bonnie-0.3.6.tar.gz/docs/about/backup-and-restore.rst Added

bonnie-0.3.6.tar.gz/docs/about/data-loss-prevention.rst Added

bonnie-0.3.6.tar.gz/docs/about/e-discovery.rst Added

bonnie-0.3.6.tar.gz/docs/architecture-and-design.rst Added

@@ -0,0 +1,1869 @@
+:tocdepth: 2
+
+.. _architecture-and-design:
+
+=======================
+Architecture and Design
+=======================
+
+The design principles could be summarized as follows:
+
+*   Eventual consistency is more important than real-time availability,
+
+*   Scalability is more important than one particular technology's individual
+    efficiency,
+
+*   Reliability of the audit trail
+
+Bonnie receives and parses :term:`event notifications` issued by Cyrus
+IMAP 2.5.
+
+An event notification is issued when a user logs on or off, and when a
+user makes a change to a mailbox or message.
+
+These event notifications are used to build the data sets for each of
+the following features included with Bonnie;
+
+    *   :ref:`about-archival`
+
+    *   :ref:`about-backup-and-restore`
+
+    *   :ref:`about-e-discovery`
+
+    *   :ref:`about-data-loss-prevention`
+
+These features are (each of them) optional, in that a deployment used
+solely for e-Discovery does not imply the inclusion of the other
+features, and most feature descriptions use super-nomenclature for
+otherwise limited, self-contained facilities that tend to serve a more
+narrow purpose.
+
+One example is a ``changelog`` module, which can be used to preserve
+selected subsets of data, such as "who changed what in an event". This
+particular module would fall under the category of e-Discovery, but not
+include the full feature-set of e-Discovery.
+
+.. rubric:: Technology Base
+
+The initial design of Bonnie is created with the following technologies
+as the basis:
+
+    *   Elasticsearch
+    *   ZeroMQ
+
+The functional components that make up a complete Bonnie environment are
+designed to be pluggable such that one or more extensions can be used
+for either of the following channels:
+
+**input**
+
+    Used to receive new event notifications, or jobs, to be handled.
+
+**storage**
+
+    Used to store event notifications, queues and/or metadata.
+
+**output**
+
+    The final and persistent record(s) of state.
+
+To parse information in to useful, digestible chunks, Bonnie uses
+so-called *handlers*, that subscribe to event notifications using
+*interests*.
+
+For example, a *Logout* event is only stored to the output channel,
+should a corresponding *Login* event be available. After all, there must
+not be a *Logout* event without a *Login* event also having occurred.
+
+A handler interested in the *Logout* event uses the storage channel to
+determine the corresponding *Login* event is indeed available.
+
+This storage channel may be configured to be an intermediate buffer,
+that holds all *Login* events until after the corresponding *Logout*
+event notification is received, or a timeout occurs, or may be
+configured to be the same channel as the output channel.
+
+Overview of Components
+======================
+
+The Bonnie infrastructure components are an add-on to an existing Cyrus
+IMAP environment, typically as part of a `Kolab Groupware`_ deployment.
+
+More detailed diagrams of communication flows in a standard Kolab
+Groupware environment are available in the
+`Kolab Groupware documentation`_, and are outside of the scope of the
+design documentation for Bonnie. Suffice it to say, that Kolab Groupware
+components communicate with Cyrus IMAP:
+
+.. graphviz::
+
+    digraph {
+            rankdir = LR;
+            splines = true;
+            overlab = prism;
+            fontname = Calibri;
+
+            edge [color=gray50, fontname=Calibri, fontsize=11];
+            node [shape=record, fontname=Calibri, fontsize=11];
+
+            "Kolab Groupware";
+
+            "Cyrus IMAP";
+
+            "Kolab Groupware" -> "Cyrus IMAP" [dir=both];
+
+        }
+
+As Bonnie is added on top of this (existing) infrastructure the design
+overview for the infrastructure becomes:
+
+.. graphviz::
+
+    digraph {
+            rankdir = LR;
+            splines = true;
+            overlab = prism;
+            fontname = Calibri;
+
+            edge [color=gray50, fontname=Calibri, fontsize=11];
+            node [shape=record, fontname=Calibri, fontsize=11];
+
+            "Kolab Groupware" [color=gray50,style=filled];
+
+            subgraph cluster_bonnie {
+                    label = "Bonnie Infrastructure";
+
+                    "Broker";
+
+                    subgraph cluster_imap {
+                            label = "IMAP Server";
+
+                            "Cyrus IMAP" [color=gray50,style=filled];
+
+                            "Collector";
+                            "Dealer";
+                        }
+
+                    "Worker";
+                }
+
+            "Kolab Groupware" -> "Cyrus IMAP" [dir=both];
+
+            "Cyrus IMAP" -> "Dealer";
+
+            "Dealer" -> "Broker";
+
+            "Broker" -> "Worker" [dir=both];
+
+            "Collector" -> "Broker" [dir=both];
+
+            "Worker" -> "Storage" [dir=both];
+
+        }
+
+The **Dealer** Component
+========================
+
+A dealer is a script executed once for each event notification, and is
+used to dispatch the event notification as fast and as efficient as
+possible.
+
+.. graphviz::
+
+    digraph {
+            rankdir = LR;
+            splines = true;
+            overlab = prism;
+            fontname = Calibri;
+
+            edge [color=gray50, fontname=Calibri, fontsize=11];
+            node [shape=record, fontname=Calibri, fontsize=11];
+
+            "Kolab Groupware" [color=gray50,style=filled];
+
+            subgraph cluster_bonnie {
+                    label = "Bonnie Infrastructure";
+
+                    "Broker";
+
+                    subgraph cluster_imap {
+                            label = "IMAP Server";
+
+                            "Cyrus IMAP" [color=gray50,style=filled];
+
+                            "Collector";
+                            "Dealer" [color="green",style=filled];
+                        }
+
+                    "Worker";
+                }

bonnie-0.3.6.tar.gz/docs/conf.py Added

@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+#
+# Bonnie documentation build configuration file, created by
+# sphinx-quickstart on Fri Nov 28 14:43:59 2014.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(1, os.path.abspath('../../pykolab.git'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = [
+        'sphinx.ext.autodoc',
+        'sphinx.ext.coverage',
+        'sphinx.ext.doctest',
+        'sphinx.ext.extlinks',
+        'sphinx.ext.graphviz',
+        'sphinx.ext.ifconfig',
+        'sphinx.ext.intersphinx',
+        'sphinx.ext.mathjax',
+        'sphinx.ext.todo',
+        'sphinx.ext.viewcode'
+    ]
+
+autodoc_member_order = 'groupwise'
+
+autodoc_default_flags = [
+        'members',
+        'undoc-members',
+        'private-members',
+        #'special-members',
+        'inherited-members',
+        'show-inheritance'
+    ]
+
+todo_include_todos = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Bonnie'
+copyright = u'2014, Jeroen van Meeuwen <vanmeeuwen@kolabsys.com>, Thomas Bruederli <bruederli@kolabsys.com>'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '0.2'
+# The full version, including alpha/beta/rc tags.
+release = '0.2.3'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinxdoc'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'Bonniedoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',

bonnie-0.3.6.tar.gz/docs/index.rst Added

bonnie-0.3.6.tar.gz/docs/modules.rst Added

bonnie-0.3.6.tar.gz/docs/technical-documentation Added

bonnie-0.3.6.tar.gz/docs/technical-documentation/daemon.rst Added

bonnie-0.3.6.tar.gz/docs/technical-documentation/index.rst Added

bonnie-0.3.6.tar.gz/docs/technical-documentation/worker-handlers.rst Added

bonnie-0.3.6.tar.gz/docs/technical-documentation/worker.rst Added

bonnie-0.3.6.tar.gz/tests Added

bonnie-0.3.6.tar.gz/tests/README.md Added

bonnie-0.3.6.tar.gz/tests/__init__.py Added

bonnie-0.3.6.tar.gz/tests/functional Added

bonnie-0.3.6.tar.gz/tests/functional/__init__.py Added

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+
+sys.path = [ '.', '../..' ] + sys.path
+
+from testbase import TestBonnieFunctional
+from test_001_login import TestBonnieLogin
+from test_002_mailboxes import TestBonnieMailboxes
+from test_003_messages import TestBonnieMessageEvents
+from test_004_changelog import TestBonnieChangelog

bonnie-0.3.6.tar.gz/tests/functional/start.sh Added

bonnie-0.3.6.tar.gz/tests/functional/stop.sh Added

bonnie-0.3.6.tar.gz/tests/functional/test_001_login.py Added

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+import time
+
+from . import TestBonnieFunctional
+from bonnie.dealer import BonnieDealer
+
+import bonnie
+conf = bonnie.getConf()
+
+class TestBonnieLogin(TestBonnieFunctional):
+
+    def test_001_login(self):
+        login = {
+            'event': 'Login',
+            'user': 'john.doe@example.org',
+            'vnd.cmu.sessionId': 'kolab-sess-test-12345',
+            'clientIP': '::1',
+            'serverDomain': 'example.org',
+        }
+
+        dealer = BonnieDealer()
+        dealer.run(json.dumps(login))
+
+        events = self.query_log([('event','=','Login')])
+        self.assertEqual(len(events), 1)
+
+        event = events[0]
+        self.assertTrue(event.has_key('user_id'))
+        self.assertTrue(event['session_id'], login['vnd.cmu.sessionId'])
+        self.assertEqual(event['@version'], bonnie.API_VERSION)
+
+        del dealer
+        time.sleep(1)
+
+        logout = {
+            'event': 'Logout',
+            'user': 'john.doe@example.org',
+            'vnd.cmu.sessionId': 'kolab-sess-test-12345',
+            'clientIP': '::1'
+        }
+
+        dealer = BonnieDealer()
+        dealer.run(json.dumps(logout))
+
+        events = self.query_log([('event','=','Login'), ('logout_time','=','*')])
+        self.assertEqual(len(events), 1)
+
+        event = events[0]
+        self.assertTrue(event.has_key('logout_time'))
+        self.assertTrue(event.has_key('duration'))
+
+        # check objects/users entry
+        user = self.storage_get(event['user_id'], index='objects', doctype='user')
+
+        self.assertIsInstance(user, dict)
+        self.assertEqual(user['user'], login['user'])
+        self.assertEqual(user['cn'], 'John Doe')

bonnie-0.3.6.tar.gz/tests/functional/test_002_mailboxes.py Added

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+
+from . import TestBonnieFunctional
+from bonnie.dealer import BonnieDealer
+
+import bonnie
+conf = bonnie.getConf()
+
+class TestBonnieMailboxes(TestBonnieFunctional):
+
+    def test_mailboxcreate(self):
+        dealer = BonnieDealer()
+
+        notification = {
+            'event': 'MailboxCreate',
+            'user': 'john.doe@example.org',
+            'uri': 'imap://john.doe@example.org@kolab.example.org/Calendar;UIDVALIDITY=12345'
+        }
+
+        dealer.run(json.dumps(notification))
+
+        events = self.query_log([('event','=','MailboxCreate')])
+        self.assertEqual(len(events), 1)
+
+        event = events[0]
+        self.assertTrue(event.has_key('folder_id'))
+        self.assertTrue(event.has_key('folder_uniqueid'))
+        self.assertTrue(event.has_key('user_id'))
+
+        # check objects/folder entry
+        folder = self.storage_get(event['folder_id'], index='objects', doctype='folder')
+
+        self.assertIsInstance(folder, dict)
+        self.assertEqual(folder['uniqueid'], event['folder_uniqueid'])
+        self.assertEqual(folder['name'], 'Calendar')
+        self.assertEqual(folder['type'], 'event')
+        self.assertEqual(folder['owner'], 'john.doe@example.org')
+
+        self.assertIsInstance(folder['metadata'], dict)
+        self.assertIsInstance(folder['acl'], dict)
+        self.assertTrue(folder['acl'].has_key(event['user_id']))
+        self.assertTrue(folder['acl'][event['user_id']].startswith('lrswi'))
+
+        # check objects/user entry
+        user = self.storage_get(event['user_id'], index='objects', doctype='user')
+
+        self.assertIsInstance(user, dict)
+        self.assertEqual(user['user'], notification['user'])
+        self.assertEqual(user['cn'], 'John Doe')

bonnie-0.3.6.tar.gz/tests/functional/test_003_messages.py Added

@@ -0,0 +1,266 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+
+from . import TestBonnieFunctional
+from bonnie.dealer import BonnieDealer
+from email import message_from_string
+
+import bonnie
+conf = bonnie.getConf()
+
+class TestBonnieMessageEvents(TestBonnieFunctional):
+
+    def test_001_messagenew(self):
+        # we assume "messageHeaders" and "messageContent" payload already being collected
+        messagenew = {
+            "event": "MessageNew",
+            "messageSize": 976,
+            "messages": 6,
+            "modseq": 20,
+            "pid": 2340,
+            "service": "lmtpunix",
+            "timestamp": "2014-10-20T13:34:14.966+02:00",
+            "uidnext": 7,
+            "uidset": "6",
+            "uri": "imap://john.doe@example.org@kolab.example.org/INBOX;UIDVALIDITY=1411487714/;UID=6",
+            "user": "john.doe@example.org",
+            "vnd.cmu.midset": [ "<a8486f5db6ec207de9b9f069850546ee@example.org>" ],
+            "vnd.cmu.sessionId": "kolab.example.org-2340-1413804854-1",
+            "vnd.cmu.unseenMessages": 3,
+            "messageHeaders": {
+                "6": {
+                    "Content-Transfer-Encoding": "7bit",
+                    "Content-Type": "text/plain",
+                    "Date": "2014-10-20T11:32:41Z",
+                    "From": [ "Br\u00fcederli, Thomas <john.doe@example.org>" ],
+                    "MIME-Version": "1.0",
+                    "Message-ID": "<a8486f5db6ec207de9b9f069850546ee@example.org>",
+                    "Received": "from kolab.example.org ([unix socket])\r\n\t by kolab.example.org (Cyrus git2.5+0-Kolab-2.5-67.el6.kolab_3.4) with LMTPA;\r\n\t Mon, 20 Oct 2014 13:34:14 +0200",
+                    "Return-Path": "<john.doe@example.org>",
+                    "Subject": "MessageNew event test",
+                    "To": [ "Doe, John <john.doe@example.org>" ],
+                    "X-Sender": "john.doe@example.org",
+                    "X-Sieve": "CMU Sieve 2.4",
+                    "X-Spam-Flag": "NO",
+                    "X-Spam-Level": "",
+                    "X-Spam-Score": "-0.002",
+                    "X-Spam-Status": "No, score=-0.002 tagged_above=-10 required=6.2\r\n\ttests=[NO_RECEIVED=-0.001, NO_RELAYS=-0.001] autolearn=ham",
+                    "X-Virus-Scanned": "amavisd-new at example.org"
+                }
+            },
+            "messageContent": {
+                "6": "Return-Path: <john.doe@example.org>\r\nReceived: from kolab.example.org ([unix socket])\r\n\t by kolab.example.org (Cyrus git2.5+0-Kolab-2.5-67.el6.kolab_3.4) with LMTPA;\r\n\t Mon, 20 Oct 2014 13:34:14 +0200\r\nX-Sieve: CMU Sieve 2.4\r\nX-Virus-Scanned: amavisd-new at example.org\r\nX-Spam-Flag: NO\r\nX-Spam-Score: -0.002\r\nX-Spam-Level: \r\nX-Spam-Status: No, score=-0.002 tagged_above=-10 required=6.2\r\n\ttests=[NO_RECEIVED=-0.001, NO_RELAYS=-0.001] autolearn=ham\r\nMIME-Version: 1.0\r\nContent-Type: text/plain; charset=US-ASCII;\r\n format=flowed\r\nContent-Transfer-Encoding: 7bit\r\nDate: Mon, 20 Oct 2014 13:32:41 +0200\r\nFrom: =?UTF-8?Q?Br=C3=BCederli=2C_Thomas?= <john.doe@example.org>\r\nTo: \"Doe, John\" <john.doe@example.org>\r\nSubject: MessageNew event test\r\nMessage-ID: <a8486f5db6ec207de9b9f069850546ee@example.org>\r\nX-Sender: john.doe@example.org\r\n\r\nThis message should trigger the MessageNew event for john.doe...\r\n...and MessageAppend to /Sent for the sender.\r\n"
+            }
+        }
+
+        dealer = BonnieDealer()
+        dealer.run(json.dumps(messagenew))
+
+        events = self.query_log([('event','=','MessageNew')])
+        self.assertEqual(len(events), 1)
+
+        event = events[0]
+        self.assertEqual(event['uidset'], '6')
+        self.assertEqual(event['service'], 'lmtpunix')
+        self.assertEqual(event['session_id'], messagenew['vnd.cmu.sessionId'])
+        self.assertEqual(event['@timestamp'], '2014-10-20T11:34:14.966000Z')
+        self.assertEqual(len(event['message_id']), 1)
+
+        self.assertIsInstance(event['headers'], dict)
+        self.assertTrue(event['headers']['Message-ID'] in event['message_id'])
+        self.assertTrue(event['headers']['Subject'] in messagenew['messageHeaders']['6']['Subject'])
+
+        # check if message payload is parsable
+        message = message_from_string(event['message'].encode('utf8','replace'))
+        self.assertEqual(message['Subject'], event['headers']['Subject'])
+
+        # check objects/folder entry
+        self.assertTrue(event.has_key('folder_id'))
+        folder = self.storage_get(event['folder_id'], index='objects', doctype='folder')
+
+        self.assertIsInstance(folder, dict)
+        self.assertEqual(folder['uniqueid'], event['folder_uniqueid'])
+        self.assertEqual(folder['name'], 'INBOX')
+        self.assertEqual(folder['owner'], 'john.doe@example.org')
+
+        # check objects/user entry
+        self.assertTrue(event.has_key('user_id'))
+        user = self.storage_get(event['user_id'], index='objects', doctype='user')
+
+        self.assertIsInstance(user, dict)
+        self.assertEqual(user['user'], messagenew['user'])
+
+
+    def test_002_messageappend(self):
+        messageappend = {
+            "event": "MessageAppend",
+            "flagNames": "\seen",
+            "messageSize": 555,
+            "messages": 6,
+            "modseq": 12,
+            "pid": 2222,
+            "service": "imap",
+            "timestamp": "2014-10-20T13:33:27.062+02:00",
+            "uidnext": 9,
+            "uidset": "8",
+            "uri": "imap://john.doe@example.org@kolab.example.org/Sent;UIDVALIDITY=1411487701/;UID=8",
+            "user": "john.doe@example.org",
+            "vnd.cmu.envelope": "(\"Mon, 20 Oct 2014 13:33:26 +0200\" \"MessageNew event test\" ((\"=?UTF-8?Q?Br=C3=BCederli=2C_Thomas?=\" NIL \"john.doe\" \"example.org\")) ((\"=?UTF-8?Q?Br=C3=BCederli=2C_Thomas?=\" NIL \"john.doe\" \"example.org\")) ((\"=?UTF-8?Q?Br=C3=BCederli=2C_Thomas?=\" NIL \"john.doe\" \"example.org\")) ((\"Doe, John\" NIL \"john.doe\" \"example.org\")) NIL NIL NIL \"<20f46a82b8584c1518fbeac7bad5f05b@example.org>\")",
+            "vnd.cmu.midset": [ "<20f46a82b8584c1518fbeac7bad5f05b@example.org>" ],
+            "vnd.cmu.sessionId": "kolab.example.org-2222-1413804806-1",
+            "vnd.cmu.unseenMessages": 0,
+            "folder_id": "76b8cd8f85bb435d17fe28d576db64a7",
+            "folder_uniqueid": "f356a1a9-f897-454f-9ada-5646fe4c4117",
+            "messageHeaders": {
+                "8": {
+                    "Content-Transfer-Encoding": "7bit",
+                    "Content-Type": "text/plain",
+                    "Date": "2014-10-20T11:31:11Z",
+                    "From": [ "Br\u00fcederli, Thomas <john.doe@example.org>" ],
+                    "MIME-Version": "1.0",
+                    "Message-ID": "<20f46a82b8584c1518fbeac7bad5f05b@example.org>",
+                    "Subject": "MessageNew event test",
+                    "To": [ "Doe, John <john.doe@example.org>" ],
+                    "User-Agent": "Kolab 3.1/Roundcube 1.1-git",
+                    "X-Sender": "john.doe@example.org"
+                }
+            },
+            "messageContent": {
+                "8": "MIME-Version: 1.0\r\nContent-Type: text/plain; charset=US-ASCII;\r\n format=flowed\r\nContent-Transfer-Encoding: 7bit\r\nDate: Mon, 20 Oct 2014 13:31:11 +0200\r\nFrom: =?UTF-8?Q?Br=C3=BCederli=2C_Thomas?= <john.doe@example.org>\r\nTo: \"Doe, John\" <john.doe@example.org>\r\nSubject: MessageNew event test\r\nMessage-ID: <44ef83beb911cb9cd82e8dc7a29467a9@example.org>\r\nX-Sender: john.doe@example.org\r\nUser-Agent: Kolab 3.1/Roundcube 1.1-git\r\n\r\nThis message should trigger the MessageNew event for john.doe...\r\n...and MessageAppend to /Sent for the sender."
+            }
+        }
+
+        dealer = BonnieDealer()
+        dealer.run(json.dumps(messageappend))
+
+        events = self.query_log([('event','=','MessageAppend')])
+        self.assertEqual(len(events), 1)
+
+        event = events[0]
+        self.assertTrue(event.has_key('user_id'))
+        self.assertIsInstance(event['headers'], dict)
+        self.assertEqual(len(event['headers']['To']), 1)
+        self.assertEqual(event['headers']['Content-Type'], 'text/plain')
+        self.assertTrue(event['headers']['Message-ID'] in event['message_id'])
+
+
+    def test_003_messageread(self):
+        messageread = {
+            "event": "MessageRead",
+            "messages": 3,
+            "modseq": 64,
+            "pid": 802,
+            "service": "imap",
+            "timestamp": "2014-10-20T13:04:09.077+02:00",
+            "uidnext": 7,
+            "uidset": "4",
+            "uri": "imap://john.doe@example.org@kolab.example.org/INBOX;UIDVALIDITY=1411487701",
+            "user": "john.doe@example.org",
+            "vnd.cmu.midset": [ "<e0ffe5d5a1569a35c1b62791390a48d5@example.org>" ],
+            "vnd.cmu.sessionId": "kolab.example.org-802-1413803049-1",
+            "vnd.cmu.unseenMessages": 0
+        }
+
+        dealer = BonnieDealer()
+        dealer.run(json.dumps(messageread))
+
+        # query by message-ID
+        events = self.query_log([('message_id','=','<e0ffe5d5a1569a35c1b62791390a48d5@example.org>')])
+        self.assertEqual(len(events), 1)
+
+        event = events[0]
+        self.assertEqual(event['uidset'], '4')
+        self.assertTrue(event.has_key('user_id'))
+        self.assertTrue(event.has_key('folder_id'))
+
+    def test_004_flagsclear(self):
+        flagsclear = {
+            "event": "FlagsClear",
+            "flagNames": "\Seen",
+            "messages": 3,

bonnie-0.3.6.tar.gz/tests/functional/test_004_changelog.py Added

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+
+from . import TestBonnieFunctional
+from bonnie.dealer import BonnieDealer
+
+import bonnie
+conf = bonnie.getConf()
+
+class TestBonnieChangelog(TestBonnieFunctional):
+
+    def test_001_changelog(self):
+        # we assume "messageHeaders" and "messageContent" payload already being collected
+        messageappend = {
+            "event": "MessageAppend",
+            "messageSize": 2932,
+            "messages": 2,
+            "modseq": 107,
+            "pid": 1248,
+            "service": "imap",
+            "timestamp": "2014-10-20T13:10:59.516+02:00",
+            "uidnext": 38,
+            "uidset": "37",
+            "uri": "imap://john.doe@example.org@kolab.example.org/Calendar;UIDVALIDITY=1411487702/;UID=37",
+            "user": "john.doe@example.org",
+            "vnd.cmu.envelope": "(\"Mon, 20 Oct 2014 13:10:59 +0200\" \"253E800C973E9FB99D174669001DB19B-FCBB6C4091F28CA0\" ((NIL NIL \"john.doe\" \"example.org\")) ((NIL NIL \"john.doe\" \"example.org\")) ((NIL NIL \"john.doe\" \"example.org\")) ((NIL NIL \"john.doe\" \"example.org\")) NIL NIL NIL NIL)",
+            "vnd.cmu.midset": [ "NIL" ],
+            "vnd.cmu.sessionId": "kolab.example.org-1248-1413803459-1",
+            "vnd.cmu.unseenMessages": 2,
+            "messageHeaders": {
+                "37": {
+                    "Content-Type": "multipart/mixed",
+                    "Date": "2014-10-20T11:23:40Z",
+                    "From": [ " <thomas.bruederli@example.org>" ],
+                    "MIME-Version": "1.0",
+                    "Subject": "253E800C973E9FB99D174669001DB19B-FCBB6C4091F28CA0",
+                    "To": [ " <thomas.bruederli@example.org>" ],
+                    "User-Agent": "Kolab 3.1/Roundcube 1.1-git",
+                    "X-Kolab-Mime-Version": "3.0",
+                    "X-Kolab-Type": "application/x-vnd.kolab.event"
+                }
+            },
+            "messageContent": {
+                "37": "MIME-Version: 1.0\r\nContent-Type: multipart/mixed;\r\n boundary=\"=_46bc539ab7a6c0a8bd4d2ddbf553df00\"\r\nFrom: thomas.bruederli@example.org\r\nTo: thomas.bruederli@example.org\r\nDate: Mon, 20 Oct 2014 13:23:40 +0200\r\nX-Kolab-Type: application/x-vnd.kolab.event\r\nX-Kolab-Mime-Version: 3.0\r\nSubject: 253E800C973E9FB99D174669001DB19B-FCBB6C4091F28CA0\r\nUser-Agent: Kolab 3.1/Roundcube 1.1-git\r\n\r\n--=_46bc539ab7a6c0a8bd4d2ddbf553df00\r\nContent-Transfer-Encoding: quoted-printable\r\nContent-Type: text/plain; charset=ISO-8859-1\r\n\r\nThis is a Kolab Groupware object....\r\n\r\n--=_46bc539ab7a6c0a8bd4d2ddbf553df00\r\nContent-Transfer-Encoding: 8bit\r\nContent-Type: application/calendar+xml; charset=UTF-8;\r\n name=kolab.xml\r\nContent-Disposition: attachment;\r\n filename=kolab.xml;\r\n size=1954\r\n\r\n<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\" ?>\r\n<icalendar xmlns=\"urn:ietf:params:xml:ns:icalendar-2.0\">\r\n\r\n  <vcalendar>\r\n    <properties>\r\n      <prodid>\r\n        <text>Roundcube-libkolab-1.1 Libkolabxml-1.1</text>\r\n      </prodid>\r\n      <version>\r\n        <text>2.0</text>\r\n      </version>\r\n      <x-kolab-version>\r\n        <text>3.1.0</text>\r\n      </x-kolab-version>\r\n    </properties>\r\n    <components>\r\n      <vevent>\r\n        <properties>\r\n          <uid>\r\n            <text>253E800C973E9FB99D174669001DB19B-FCBB6C4091F28CA0</text>\r\n          </uid>\r\n          <created>\r\n            <date-time>2014-09-23T23:31:23Z</date-time>\r\n          </created>\r\n          <dtstamp>\r\n            <date-time>2014-10-20T11:23:40Z</date-time>\r\n          </dtstamp>\r\n          <sequence>\r\n            <integer>28</integer>\r\n          </sequence>\r\n          <class>\r\n            <text>PUBLIC</text>\r\n          </class>\r\n          <dtstart>\r\n            <parameters>\r\n              <tzid>\r\n                <text>/kolab.org/Europe/Berlin</text>\r\n              </tzid>\r\n            </parameters>\r\n            <date-time>2014-10-20T14:00:00</date-time>\r\n          </dtstart>\r\n          <dtend>\r\n            <parameters>\r\n              <tzid>\r\n                <text>/kolab.org/Europe/Berlin</text>\r\n              </tzid>\r\n            </parameters>\r\n            <date-time>2014-10-20T16:00:00</date-time>\r\n          </dtend>\r\n          <summary>\r\n            <text>Today</text>\r\n          </summary>\r\n          <description>\r\n            <text>(new revision)</text>\r\n          </description>\r\n          <organizer>\r\n            <parameters>\r\n              <cn>\r\n                <text>Br\u00fcederli, Thomas</text>\r\n              </cn>\r\n            </parameters>\r\n            <cal-address>mailto:%3Cthomas.bruederli%40example.org%3E</cal-address>\r\n          </organizer>\r\n        </properties>\r\n      </vevent>\r\n    </components>\r\n  </vcalendar>\r\n\r\n</icalendar>\r\n\r\n--=_46bc539ab7a6c0a8bd4d2ddbf553df00--\r\n"
+            }
+        }
+
+        dealer = BonnieDealer()
+        dealer.run(json.dumps(messageappend))
+
+        # query by subject (i.e. object UUID)
+        events = self.query_log([('headers.Subject','=','253E800C973E9FB99D174669001DB19B-FCBB6C4091F28CA0')])
+        self.assertEqual(len(events), 1)
+
+        event = events[0]
+        self.assertIsInstance(event['headers'], dict)
+        self.assertEqual(event['headers']['X-Kolab-Mime-Version'], '3.0')
+        self.assertEqual(event['headers']['X-Kolab-Type'], 'application/x-vnd.kolab.event')
+
+        self.assertTrue(event.has_key('user_id'))
+        self.assertTrue(event.has_key('revision'))
+

bonnie-0.3.6.tar.gz/tests/functional/testbase.py Added

@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import json
+import time
+
+from twisted.trial import unittest
+from subprocess import call
+from bonnie.dealer import BonnieDealer
+from bonnie.worker.storage import ElasticSearchStorage
+
+pwd = os.path.dirname(__file__)
+basedir = os.path.join(pwd, '..', '..')
+
+import bonnie
+conf = bonnie.getConf()
+conf.finalize_conf()
+
+class TestBonnieFunctional(unittest.TestCase):
+    attempts = 12
+
+    def setUp(self):
+        self.storage = ElasticSearchStorage()
+        self.storage.es.indices.delete(index='logstash-*', ignore=[400, 404])
+        self.storage.es.indices.delete(index='objects', ignore=[400, 404])
+
+        call([os.path.join(pwd, 'start.sh'), basedir])
+        time.sleep(1)
+
+    def tearDown(self):
+        call([os.path.join(pwd, 'stop.sh')])
+        time.sleep(2)
+
+    def query_log(self, query):
+        attempts = self.attempts
+        while attempts > 0:
+            attempts -= 1
+            res = self.storage.select(query, index='logstash-*', doctype='logs', sortby='@timestamp:desc')
+            if res and res['total'] > 0:
+                return res['hits']
+
+            time.sleep(1)
+            # print "query retry", attempts
+
+        return None
+
+    def storage_get(self, key, index, doctype):
+        attempts = self.attempts
+        while attempts > 0:
+            attempts -= 1
+            res = self.storage.get(key, index=index, doctype=doctype)
+            if res is not None:
+                return res
+            time.sleep(1)
+            # print "get retry", attempts
+
+        return None
+

bonnie-0.3.6.tar.gz/tests/unit Added

bonnie-0.3.6.tar.gz/tests/unit/__init__.py Added

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+
+sys.path = [ '.', '../..' ] + sys.path
+
+from test_001_utils import TestBonnieUtils
+from test_002_collector import TestBonnieCollector
+from test_003_dealer import TestBonnieDealer
+from test_004_worker import TestBonnieWorker
+from test_005_persistence import TestBonniePersistence
+from test_006_caching import TestBonnieCaching
+

bonnie-0.3.6.tar.gz/tests/unit/resources Added

bonnie-0.3.6.tar.gz/tests/unit/resources/3. Added

bonnie-0.3.6.tar.gz/tests/unit/resources/event_mime_message.eml Added

@@ -0,0 +1,95 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+ boundary="=_ddac4015cd2f0b6b232ed6f53d8bd55b"
+From: john.doe@example.org
+To: john.doe@example.org
+Date: Wed, 24 Sep 2014 06:52:00 +0200
+X-Kolab-Type: application/x-vnd.kolab.event
+X-Kolab-Mime-Version: 3.0
+Subject: 253E800C973E9FB99D174669001DB19B-FCBB6C4091F28CA0
+User-Agent: Kolab 3.1/Roundcube 1.1-git
+
+--=_ddac4015cd2f0b6b232ed6f53d8bd55b
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a Kolab Groupware object. To view this object you will need an emai=
+l client that understands the Kolab Groupware format. For a list of such em=
+ail clients please visit http://www.kolab.org/
+
+
+--=_ddac4015cd2f0b6b232ed6f53d8bd55b
+Content-Transfer-Encoding: 8bit
+Content-Type: application/calendar+xml; charset=UTF-8; name=kolab.xml
+Content-Disposition: attachment;
+ filename=kolab.xml;
+ size=1864
+
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<icalendar xmlns="urn:ietf:params:xml:ns:icalendar-2.0">
+
+  <vcalendar>
+    <properties>
+      <prodid>
+        <text>Roundcube-libkolab-1.1 Libkolabxml-1.1</text>
+      </prodid>
+      <version>
+        <text>2.0</text>
+      </version>
+      <x-kolab-version>
+        <text>3.1.0</text>
+      </x-kolab-version>
+    </properties>
+    <components>
+      <vevent>
+        <properties>
+          <uid>
+            <text>253E800C973E9FB99D174669001DB19B-FCBB6C4091F28CA0</text>
+          </uid>
+          <created>
+            <date-time>2014-09-23T23:31:23Z</date-time>
+          </created>
+          <dtstamp>
+            <date-time>2014-09-24T04:52:00Z</date-time>
+          </dtstamp>
+          <sequence>
+            <integer>3</integer>
+          </sequence>
+          <class>
+            <text>PUBLIC</text>
+          </class>
+          <dtstart>
+            <parameters>
+              <tzid>
+                <text>/kolab.org/Europe/Berlin</text>
+              </tzid>
+            </parameters>
+            <date-time>2014-09-26T14:00:00</date-time>
+          </dtstart>
+          <dtend>
+            <parameters>
+              <tzid>
+                <text>/kolab.org/Europe/Berlin</text>
+              </tzid>
+            </parameters>
+            <date-time>2014-09-26T15:00:00</date-time>
+          </dtend>
+          <summary>
+            <text>Test</text>
+          </summary>
+          <organizer>
+            <parameters>
+              <cn>
+                <text>Doe, John</text>
+              </cn>
+            </parameters>
+            <cal-address>mailto:%3Cjohn.doe%40example.org%3E</cal-address>
+          </organizer>
+        </properties>
+      </vevent>
+    </components>
+  </vcalendar>
+
+</icalendar>
+
+--=_ddac4015cd2f0b6b232ed6f53d8bd55b--
\ No newline at end of file

bonnie-0.3.6.tar.gz/tests/unit/test_001_utils.py Added

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+from bonnie.utils import expand_uidset
+from bonnie.utils import parse_imap_uri
+from bonnie.utils import mail_message2dict
+from bonnie.utils import decode_message_headers
+from bonnie.utils import imap_folder_path
+from bonnie.utils import imap_mailbox_fs_path
+from email import message_from_string
+from twisted.trial import unittest
+
+
+class TestBonnieUtils(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def _get_resource(self, filename):
+        pwd = os.path.dirname(__file__)
+        filepath = os.path.join(pwd, 'resources', filename)
+        fp = open(filepath, 'r')
+        data = fp.read()
+        fp.close()
+        return data
+
+    def test_expand_uidset(self):
+        self.assertEqual(expand_uidset('3'), ['3'])
+        self.assertEqual(expand_uidset('3,5'), ['3', '5'])
+        self.assertEqual(expand_uidset('3:5'), ['3', '4', '5'])
+
+    def test_parse_imap_uri(self):
+        url = parse_imap_uri(
+            "imap://john.doe@example.org@kolab.example.org/Calendar/Personal%20Calendar;UIDVALIDITY=1411487702/;UID=3"
+        )
+
+        self.assertEqual(url['host'],   'kolab.example.org')
+        self.assertEqual(url['user'],   'john.doe')
+        self.assertEqual(url['domain'], 'example.org')
+        self.assertEqual(url['path'],   'Calendar/Personal Calendar')
+        self.assertEqual(url['UID'],    '3')
+
+    def test_decode_message_headers(self):
+        message = message_from_string(self._get_resource('3.'))
+        headers = decode_message_headers(message)
+
+        self.assertEqual(len(headers['From']), 1)
+        self.assertEqual(len(headers['To']), 2)
+        self.assertEqual(
+            headers['To'][0],
+            u'Br\u00fcderli, Thomas <thomas.bruederli@example.org>'
+        )
+
+        self.assertEqual(headers['Content-Type'], 'text/plain')
+        self.assertEqual(headers['Date'], '2014-09-24T04:52:00Z')
+        self.assertEqual(headers['Subject'], 'Test')
+
+    def test_mail_message2dict(self):
+        message = mail_message2dict(
+            self._get_resource('event_mime_message.eml')
+        )
+
+        self.assertIsInstance(message, dict)
+        self.assertEqual(
+            message['Subject'],
+            '253E800C973E9FB99D174669001DB19B-FCBB6C4091F28CA0'
+        )
+
+        self.assertEqual(
+            message['X-Kolab-Type'],
+            'application/x-vnd.kolab.event'
+        )
+
+        self.assertEqual(len(message['@parts']), 2)
+
+        xmlpart = message['@parts'][1]
+        self.assertEqual(
+            xmlpart['Content-Type'],
+            'application/calendar+xml; charset=UTF-8; name=kolab.xml'
+        )
+
+        message2 = mail_message2dict("FOO")
+        self.assertIsInstance(message2, dict)
+        self.assertEqual(message2['@body'], "FOO")
+
+    def test_imap_folder_path(self):
+        p1 = imap_folder_path(
+            "imap://john.doe@example.org@kolab.example.org/Calendar;UID=3"
+        )
+
+        self.assertEqual(p1, "user/john.doe/Calendar@example.org")
+
+        p2 = imap_folder_path(
+            "imap://john.doe@example.org@kolab.example.org/INBOX;UIDVALIDITY=1411487702"
+        )
+
+        self.assertEqual(p2, "user/john.doe@example.org")
+
+        # test shared folders (but how are they referred in the uri?)
+        p3 = imap_folder_path(
+            "imap://kolab.example.org/Shared%20Folders/shared/Project-X%40example.org;UIDVALIDITY=1412093781/;UID=2"
+        )
+
+        self.assertEqual(p3, "shared/Project-X@example.org")
+
+    def test_imap_mailbox_fs_path(self):
+        path = imap_mailbox_fs_path(
+            "imap://john.doe@example.org@kolab.example.org/Calendar/Personal%20Calendar;UID=3"
+        )
+
+        self.assertEqual(
+            path,
+            "/var/spool/imap/domain/e/example.org/j/user/john^doe/Calendar/Personal Calendar"
+        )

bonnie-0.3.6.tar.gz/tests/unit/test_002_collector.py Added

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import json
+import bonnie
+from bonnie.collector import BonnieCollector
+from bonnie.collector.handlers import MessageDataHandler, IMAPDataHandler
+from twisted.trial import unittest
+
+
+class TestBonnieCollector(unittest.TestCase):
+
+    def setUp(self):
+        # patch bonnie.utils.imap_mailbox_fs_path() to point to local resources folder
+        self.patch(bonnie.utils, 'imap_mailbox_fs_path', self._imap_mailbox_fs_path)
+
+    def _imap_mailbox_fs_path(self, uri):
+        pwd = os.path.dirname(__file__)
+        return os.path.join(pwd, 'resources')
+
+    def test_retrieve_headers_for_messages(self):
+        coll = MessageDataHandler()
+        notification = { 'uri': 'imap://john.doe@example.org@kolab.example.org/Calendar', 'uidset': '3' }
+        notification = json.loads(coll.retrieve_headers_for_messages(json.dumps(notification)))
+
+        self.assertTrue(notification.has_key('messageHeaders'))
+        self.assertIsInstance(notification['messageHeaders'], dict)
+        self.assertTrue(notification['messageHeaders'].has_key('3'))
+
+        headers = notification['messageHeaders']['3']
+        self.assertIsInstance(headers, dict)
+        self.assertEqual(headers['Subject'], 'Test')
+        self.assertEqual(headers['From'][0], 'John Doe <john.doe@example.org>')
+
+    def test_retrieve_contents_for_messages(self):
+        coll = MessageDataHandler()
+        notification = { 'uri': 'imap://john.doe@example.org@kolab.example.org/Calendar', 'uidset': '3' }
+        notification = json.loads(coll.retrieve_contents_for_messages(json.dumps(notification)))
+
+        self.assertTrue(notification.has_key('messageContent'))
+        self.assertIsInstance(notification['messageContent'], dict)
+        self.assertTrue(notification['messageContent'].has_key('3'))
+        self.assertIsInstance(notification['messageContent']['3'], unicode)
+
+        self.assertTrue(notification.has_key('messageHeaders'))
+        self.assertIsInstance(notification['messageHeaders'], dict)
+        self.assertTrue(notification['messageHeaders'].has_key('3'))
+        self.assertIsInstance(notification['messageHeaders']['3'], dict)
+
+    def test_get_imap_folder_metadata(self):
+        coll = IMAPDataHandler()
+        notification = { 'uri': 'imap://john.doe@example.org@kolab.example.org/Calendar' }
+        notification = json.loads(coll.get_imap_folder_metadata(json.dumps(notification)))
+
+        self.assertTrue(notification.has_key('metadata'))
+        self.assertIsInstance(notification['metadata'], dict)
+        self.assertEqual(notification['metadata']['/shared/vendor/kolab/folder-type'], 'event')
+
+    def test_get_imap_folder_acl(self):
+        coll = IMAPDataHandler()
+        notification = { 'uri': 'imap://john.doe@example.org@kolab.example.org/Calendar' }
+        notification = json.loads(coll.get_imap_folder_acl(json.dumps(notification)))
+
+        self.assertTrue(notification.has_key('acl'))
+        self.assertIsInstance(notification['acl'], dict)
+        self.assertEqual(notification['acl']['john.doe@example.org'], 'lrswipkxtecdan')
+
+    def test_zz_execute(self):
+        coll = BonnieCollector()
+
+        notification = { 'uri': 'imap://john.doe@example.org@kolab.example.org/Calendar;UID=3' }
+        result = json.loads(coll.execute('FETCH', json.dumps(notification)))
+
+        self.assertTrue(result.has_key('messageContent'))
+        self.assertTrue(result.has_key('uidset'))
+        self.assertEqual(result['uidset'], '3')
+
+        notification['vnd.cmu.oldUidset'] = '3'
+        result = json.loads(coll.execute('HEADER', json.dumps(notification)))
+        self.assertTrue(result.has_key('messageHeaders'))
+
+        result = json.loads(coll.execute('GETMETADATA', json.dumps(notification)))
+        self.assertTrue(result.has_key('metadata'))
+
+        result = json.loads(coll.execute('GETACL', json.dumps(notification)))
+        self.assertTrue(result.has_key('acl'))

bonnie-0.3.6.tar.gz/tests/unit/test_003_dealer.py Added

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import json
+import bonnie
+from bonnie.dealer import BonnieDealer
+from twisted.trial import unittest
+
+class BonnieDealerOutput(object):
+    """
+        Mockup class to catch event notifications forwarded by BonnieDealer
+    """
+    output_calls = []
+
+    def __init__(self, *args, **kw):
+        pass
+
+    def name(self):
+        return 'zmq_output'
+
+    def register(self, *args, **kw):
+        pass
+
+    def run(self, notification):
+        notification = json.loads(notification)
+        self.output_calls.append(notification['event'])
+
+class TestBonnieDealer(unittest.TestCase):
+
+    def setUp(self):
+        # patch bonnie.outputs.list_classes() to register a dummy output handler
+        self.patch(bonnie.dealer.outputs, 'list_classes', self._outputs_list_classes)
+
+    def _outputs_list_classes(self):
+        return [ BonnieDealerOutput ]
+
+    def test_run(self):
+        dealer = BonnieDealer()
+        dealer.run(json.dumps({ 'event':'Login', 'user':'john.doe@example.org' }))
+        self.assertIn('Login', BonnieDealerOutput.output_calls)
+
+        # don't trigger blacklisted events
+        BonnieDealerOutput.output_calls = []
+        dealer.run(json.dumps({ 'event':'Login', 'user':'cyrus-admin' }))
+        self.assertEqual(len(BonnieDealerOutput.output_calls), 0)
+
+        dealer.run(json.dumps({ 'event':'MessageAppend', 'user':'cyrus-admin' }))
+        self.assertEqual(len(BonnieDealerOutput.output_calls), 1)
+        self.assertIn('MessageAppend', BonnieDealerOutput.output_calls)

bonnie-0.3.6.tar.gz/tests/unit/test_004_worker.py Added

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+import elasticsearch
+import bonnie
+
+from bonnie.worker.storage import ElasticSearchStorage
+from twisted.trial import unittest
+
+class MockupElasticsearch(object):
+    """
+        Dummy mockup to avoid real connections to ES
+    """
+
+    def __init__(self, *args, **kw):
+        pass
+
+    def search(self, *args, **kw):
+        pass
+
+    def create(self, *args, **kw):
+        pass
+
+    def update(self, *args, **kw):
+        pass
+
+
+class TestBonnieWorker(unittest.TestCase):
+
+    def setUp(self):
+        self.patch(elasticsearch, 'Elasticsearch', MockupElasticsearch)
+
+    def test_001_notificaton2folder(self):
+        notification = {
+            'event': 'MailboxCreate',
+            'uri': 'imap://john.doe@example.org@kolab.example.org/Calendar/Subcal;UIDVALIDITY=12345',
+            'metadata': {
+                '/shared/vendor/cmu/cyrus-imapd/partition':          'default',
+                '/shared/vendor/cmu/cyrus-imapd/lastupdate':         '23-Sep-2014 11:55:15 -0400',
+                '/shared/vendor/cmu/cyrus-imapd/duplicatedeliver':   'false',
+                '/shared/vendor/cmu/cyrus-imapd/pop3newuidl':        'true',
+                '/shared/vendor/cmu/cyrus-imapd/size':               '0',
+                '/shared/vendor/cmu/cyrus-imapd/sharedseen':         'false',
+                '/shared/vendor/kolab/folder-type':                  'event',
+                '/shared/vendor/cmu/cyrus-imapd/uniqueid':           '28614de9-3614-422d-be5d-5716605ef0fc',
+            },
+            'acl': {
+                'john.doe@example.org': "lrswipkxtecdan",
+                'anyone': 'lrs',
+            }
+        }
+        storage = ElasticSearchStorage()
+        folder = storage.notificaton2folder(notification)
+        folder_id = folder['id']
+
+        self.assertTrue(folder.has_key('body'))
+        self.assertEqual(folder['body']['uniqueid'], notification['folder_uniqueid'])
+        self.assertEqual(folder['body']['server'], 'kolab.example.org')
+        self.assertEqual(folder['body']['owner'], 'john.doe@example.org')
+        self.assertEqual(folder['body']['uri'], 'imap://john.doe@example.org@kolab.example.org/Calendar/Subcal')
+
+        # check changes in ignored metadata
+        notification['metadata']['/shared/vendor/cmu/cyrus-imapd/lastupdate'] = '24-Sep-2014 16:21:19 -0400'
+        notification['metadata']['/shared/vendor/cmu/cyrus-imapd/size'] = '88'
+        notification['acl'] = {
+            'anyone': 'lrs',
+            'john.doe@example.org': "lrswipkxtecdan",
+        }
+        notification['uri'] = 'imap://john.doe@example.org@kolab.example.org/Calendar/RENAMED;UIDVALIDITY=123456'
+        folder = storage.notificaton2folder(notification)
+        self.assertEqual(folder['id'], folder_id)
+
+        # detect ACL changes
+        notification['acl'] = {
+            'john.doe@example.org': "lrswipkxtecdan",
+            'anyone': 'lrswd',
+        }
+        folder = storage.notificaton2folder(notification)
+        self.assertNotEqual(folder['id'], folder_id)

bonnie-0.3.6.tar.gz/tests/unit/test_005_persistence.py Added

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import json
+import bonnie
+from bonnie.broker import persistence
+from bonnie.broker.persistence import db, PersistentBase
+from bonnie.broker.brokers.zmq_broker.job import Job
+from twisted.trial import unittest
+
+class PlistItem(PersistentBase):
+    __tablename__ = 'plisttest'
+    id = db.Column(db.Integer, primary_key=True)
+    value = db.Column(db.String)
+
+    def __init__(self, value, id=None):
+        self.value = value
+        self.id = id
+
+    def __repr__(self):
+        return '<PlistItem %s:%r>' % (self.id, self.value)
+
+
+class TestBonniePersistence(unittest.TestCase):
+
+    def setUp(self):
+        pass
+
+    def tearDown(self):
+        # TODO: clear database
+        # PersistentBase.metadata.drop_all(bonnie.broker.persistence.engine)
+        pass
+
+    def test_001_base_list(self):
+        plist = persistence.List('base', PlistItem)
+        plist.append(PlistItem("One"))
+        plist.append(PlistItem("Two"))
+        item3 = PlistItem("Three")
+        plist.append(item3)
+
+        self.assertEqual(len(plist), 3)
+        self.assertEqual(plist[2], item3)
+        self.assertEqual(2, plist.index(item3))
+        self.assertTrue(item3 in plist)
+
+        plist.append(PlistItem("Five"))
+        plist.append(PlistItem("Six"))
+
+        plist[4] = PlistItem("Five.5")
+        self.assertEqual(plist[4].value, "Five.5")
+
+        del plist[plist.index(item3)]
+        plist.pop(3)
+        plist.pop()
+        self.assertEqual(len(plist), 2)
+
+        i = 0
+        for item in plist:
+            i += 1
+            self.assertTrue(isinstance(item, PlistItem))
+
+        self.assertEqual(i, 2)
+
+
+    def test_003_broker_jobs(self):
+        worker_jobs = persistence.List('worker', Job)
+        collector_jobs = persistence.List('collector', Job)
+        one = Job(state='PENDING', notification='{"state":"pending","event":"test"}', collector_id='C.1')
+        two = Job(state='PENDING', notification='{"state":"pending","event":"other"}', collector_id='C.1')
+        done = Job(state='DONE', notification='{"state":"done","event":"passed"}', collector_id='C.1')
+        worker_jobs.append(one)
+        worker_jobs.append(two)
+        worker_jobs.append(done)
+
+        self.assertEqual(len(worker_jobs), 3)
+        self.assertEqual(len(collector_jobs), 0)
+        self.assertTrue(one.uuid in [x.uuid for x in worker_jobs])
+        pending = [x for x in worker_jobs if x.state == 'PENDING' and x.collector_id == 'C.1']
+        self.assertEqual(len(pending), 2)
+        job = pending.pop()
+
+        self.assertEqual(job, two)
+        self.assertEqual(len(pending), 1)
+        self.assertEqual(len(worker_jobs), 3)
+
+        # move job to another list
+        collector_jobs.append(one)
+        self.assertTrue(one in collector_jobs)
+        self.assertTrue(one in worker_jobs)
+
+        worker_jobs.remove(one)
+        self.assertFalse(one in worker_jobs)
+
+        self.assertEqual(len(worker_jobs), 2)
+        self.assertEqual(len(collector_jobs), 1)
+
+        # move job to the end of the queue
+        worker_jobs.remove(two)
+        worker_jobs.append(two)
+
+        self.assertEqual(worker_jobs[0], done)
+        self.assertEqual(worker_jobs[-1], two)

bonnie-0.3.6.tar.gz/tests/unit/test_006_caching.py Added

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+
+from twisted.trial import unittest
+from bonnie.worker.storage import CachedDict
+
+class TestBonnieCaching(unittest.TestCase):
+
+    def test_cached_dict(self):
+        # dict with 10 seconds TTL
+        d = CachedDict(10)
+        d['one'] = 'ONE'
+        d['two'] = 'TWO'
+
+        self.assertEqual(len(d), 2)
+        self.assertTrue(d.has_key('one'))
+        self.assertFalse(d.has_key('none'))
+        self.assertEqual(d['one'], 'ONE')
+        # internal sorting is influenced by the expiry time
+        # but sorting in dicts is irrelevant in most cases
+        self.assertEqual(sorted(d.keys()), ['one','two'])
+        self.assertEqual(sorted(d.values()), ['ONE','TWO'])
+        self.assertEqual(sorted(d.items()), [ ('one','ONE'), ('two','TWO') ])
+
+        time.sleep(5)
+        d['five'] = 'FIVE'
+        time.sleep(1)
+        d['six'] = 'SIX'
+        self.assertEqual(len(d), 4)
+        self.assertEqual(d.pop('five'), 'FIVE')
+        self.assertEqual(len(d), 3)
+
+        # let the first two items expire
+        time.sleep(5)
+        self.assertEqual(len(d), 1)
+        self.assertFalse(d.has_key('one'))
+
+        # test iterator
+        # 'five' was popped, thus only 'six' remains
+        for k,v in d:
+            self.assertEqual(k, 'six')
+            self.assertEqual(v, 'SIX')
+
+        # all expired
+        time.sleep(5)
+        self.assertEqual(len(d), 0)
+        self.assertEqual(len(d.data), 3)
+
+        # expunge internal cache
+        d.expunge()
+        self.assertEqual(len(d.data), 0)

bonnie-0.3.6.tar.gz/worker.py Added

@@ -0,0 +1,27 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+# Copyright 2010-2014 Kolab Systems AG (http://www.kolabsys.com)
+#
+# Jeroen van Meeuwen (Kolab Systems) <vanmeeuwen a kolabsys.com>
+# Thomas Bruederli (Kolab Systems) <bruederli a kolabsys.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+from bonnie.worker import BonnieWorker
+
+if __name__ == "__main__":
+    worker = BonnieWorker()
+    worker.start()
+