Kolab Groupware OBS

We truncated the diff of some files because they were too big. If you want to see the full diff for every file, click here.

Changes of Revision 3

egara.spec Changed

@@ -12,8 +12,8 @@
 %global mail_group_id 76
 
 Name:               egara
-Version: 0.2
-Release: 0.20150708.git%{?dist}
+Version:            0.2
+Release:            0.20161121.git%{?dist}
 Summary:            Event driven groupware archival system
 
 Group:              System Environment/Daemons
@@ -23,41 +23,46 @@
 # From 3e4a3da61124e9c79b7f7f49516e6e86aa072051
 Source0:            http://mirror.kolabsys.com/pub/releases/%{name}-%{version}.tar.gz
 
-Patch1:             egara-0.1-tests.patch
-Patch2:             egara-0.1-release.patch
-Patch3:             egara-0.2-relax-riakc-dep.patch
+Patch1:             make-it-very-easy-for-rebar3.patch
 
 BuildRequires:      erlang
+BuildRequires:      erlang-eimap >= 0.4.0
 BuildRequires:      erlang-eldap
 BuildRequires:      erlang-goldrush >= 0.1.6
 BuildRequires:      erlang-inert
 BuildRequires:      erlang-iso8601
 BuildRequires:      erlang-jsx >= 2.4.0
 BuildRequires:      erlang-lager >= 2.1.0
+BuildRequires:      erlang-lager_syslog
 BuildRequires:      erlang-meck >= 0.8.2
 BuildRequires:      erlang-poolboy >= 1.4.2
 BuildRequires:      erlang-procket
 BuildRequires:      erlang-protobuffs
-BuildRequires:      erlang-rebar
+BuildRequires:      erlang-rebar3 >= 3.3.2
 BuildRequires:      erlang-riak_pb
 BuildRequires:      erlang-riakc >= 2.0.1
+BuildRequires:      erlang-rpm-macros
+BuildRequires:      erlang-syn >= 1.5.0
 
 Requires(pre):      shadow-utils
 Requires(postun):   shadow-utils
 
 Requires:           erlang
+Requires:           erlang-eimap >= 0.4.0
 Requires:           erlang-eldap
 Requires:           erlang-goldrush >= 0.1.6
 Requires:           erlang-inert
 Requires:           erlang-iso8601
 Requires:           erlang-jsx => 2.4.0
 Requires:           erlang-lager >= 2.1.0
+Requires:           erlang-lager_syslog
 Requires:           erlang-meck >= 0.8.2
 Requires:           erlang-poolboy >= 1.4.2
 Requires:           erlang-procket
 Requires:           erlang-protobuffs
 Requires:           erlang-riak_pb
 Requires:           erlang-riakc >= 2.0.1
+Requires:           erlang-syn >= 1.5.0
 
 %description
 Egara is an event driven groupware archival system for e-Discovery
@@ -68,45 +73,74 @@
 
 %prep
 %setup -q
+
 %patch1 -p1
-%patch2 -p1
-%patch3 -p1
 
 %build
-pushd rel
-rebar create-node nodeid=egara
-popd
-ENABLE_STATIC=no rebar compile -v
-rebar generate -v
+DEBUG=1
+export DEBUG
+
+HEX_OFFLINE=true
+export HEX_OFFLINE
+
+rebar3 release \
+    --dev-mode false \
+    --relname %{name} \
+    --relvsn %{version} \
+    --verbose
 
 %install
-mkdir -p %{buildroot}%{_libdir}/erlang/lib/%{realname}-%{version}/ebin
-install -m 644 apps/egara/ebin/%{realname}.app %{buildroot}%{_libdir}/erlang/lib/%{realname}-%{version}/ebin
-install -m 644 apps/egara/ebin/*.beam %{buildroot}%{_libdir}/erlang/lib/%{realname}-%{version}/ebin
+
+find . -type f | sort
+
+mkdir -p \
+    %{buildroot}%{_sysconfdir}/%{name}/ \
+    %{buildroot}%{_sbindir} \
+    %{buildroot}%{_erldir}/bin/ \
+    %{buildroot}%{_erllibdir}/%{realname}-%{version}/
+
+install app.config %{buildroot}%{_sysconfdir}/%{name}/sys.config
+
+pushd %{buildroot}%{_erldir}/bin/
+ln -s ../lib/%{name}-%{version}/bin/egara egara
+popd
+
+cp -a _build/default/rel/egara/bin/ %{buildroot}%{_erllibdir}/%{realname}-%{version}/
+cp -a _build/default/rel/egara/lib/egara-%{version}/ebin/ %{buildroot}%{_erllibdir}/%{realname}-%{version}/
+cp -a _build/default/rel/egara/releases %{buildroot}%{_erllibdir}/%{realname}-%{version}/
+
+pushd %{buildroot}%{_libdir}/erlang/lib/%{realname}-%{version}/releases/%{version}/
+ln -sfv ../../../../../../..%{_sysconfdir}/%{name}/sys.config sys.config
+mv vm.args ../../../../../../..%{_sysconfdir}/%{name}/vm.args
+ln -sv ../../../../../../..%{_sysconfdir}/%{name}/vm.args vm.args
+popd
+
+pushd %{buildroot}%{_libdir}/erlang/lib/%{realname}-%{version}/
+ln -s ../../lib lib
+popd
+
+pushd %{buildroot}%{_sbindir}
+ln -sfv ../..%{_erllibdir}/%{name}-%{version}/bin/egara egara
+popd
 
 %check
-rebar skip_deps=true eunit -v
-
-%pre
-if [ $1 == 1 ]; then
-    /usr/sbin/groupadd -g %{mail_group_id} --system %{mail_group} 2> /dev/null || :
-    /usr/sbin/groupadd --system %{egara_group} 2> /dev/null || :
-    /usr/sbin/useradd -c "Egara Service" -d %{_var}/lib/egara -g %{egara_group} \
-        -G %{mail_group} -s /sbin/nologin --system %{egara_user} 2> /dev/null || :
-fi
-
-%postun
-if [ $1 == 0 ]; then
-    /usr/sbin/userdel %{egara_user} 2>/dev/null || :
-    /usr/sbin/groupdel %{egara_group} 2>/dev/null || :
-fi
+DEBUG=1
+export DEBUG
+
+HEX_OFFLINE=true
+export HEX_OFFLINE
+
+# Hopeless for -0.2
+rebar3 eunit -v || :
 
 %files
 %doc README.md
-%dir %{_libdir}/erlang/lib/%{realname}-%{version}
-%dir %{_libdir}/erlang/lib/%{realname}-%{version}/ebin
-%{_libdir}/erlang/lib/%{realname}-%{version}/ebin/%{realname}.app
-%{_libdir}/erlang/lib/%{realname}-%{version}/ebin/*.beam
+%{_sbindir}/egara
+%dir %{_sysconfdir}/egara
+%config(noreplace) %{_sysconfdir}/egara/sys.config
+%config(noreplace) %{_sysconfdir}/egara/vm.args
+%{_erldir}/bin/*
+%{_erllibdir}/%{realname}-%{version}
 
 %changelog
 * Fri May 15 2015 Jeroen van Meeuwen <vanmeeuwen@kolabsys.com> - 0.1-0.1.git

make-it-very-easy-for-rebar3.patch Added

@@ -0,0 +1,60 @@
+diff -ur egara-0.2.orig/apps/egara/src/egara.app.src egara-0.2/apps/egara/src/egara.app.src
+--- egara-0.2.orig/apps/egara/src/egara.app.src	2016-11-11 15:52:50.000000000 +0100
++++ egara-0.2/apps/egara/src/egara.app.src	2016-11-21 14:49:55.300535578 +0100
+@@ -2,24 +2,14 @@
+ {application, egara,
+  [
+   {description, "Kolab event and groupware data archival, e-discovery and data loss prevention system."},
+-  {vsn, "0.1"},
++  {vsn, "0.2"},
+   {registered, []},
+   {applications, [
+                   kernel,
+                   stdlib,
+                   crypto,
+                   compiler,
+-                  syntax_tools,
+-                  goldrush,
+-                  lager,
+-                  jsx,
+-                  procket,
+-                  inert,
+-                  mnesia,
+-                  poolboy,
+-                  riakc,
+-                  syn,
+-                  eimap
++                  syntax_tools
+                  ]},
+   {mod, { egara, []}},
+   {env, [
+Only in egara-0.2: _build
+diff -ur egara-0.2.orig/rebar.config egara-0.2/rebar.config
+--- egara-0.2.orig/rebar.config	2016-11-11 15:52:50.000000000 +0100
++++ egara-0.2/rebar.config	2016-11-21 14:45:37.371742363 +0100
+@@ -9,23 +9,9 @@
+             ]
+ }.
+ 
+-{ deps, [
+-    lager,
+-    jsx,
+-    { lager_syslog, "3.*", { git, "git://github.com/basho/lager_syslog.git", { tag, "3.0.1" } } },
+-    { inert, ".*", { git, "https://github.com/msantos/inert.git", { branch, "master" } } }, %{ tag, "0.6.2" } } },
+-    { procket, ".*", { git, "https://github.com/msantos/procket.git", { branch, "master" } } }, %{ tag, "0.7.0" } } },
+-    { riakc, ".*", { git, "https://github.com/basho/riak-erlang-client", { tag, "2.5.0" } } },
+-    poolboy,
+-    {'iso8601', ".*", { git, "https://github.com/erlsci/iso8601.git", { tag, "1.2.1" } } },
+-    { syn, "1.5.0" },
+-    { eimap, ".*", { git, "https://git.kolab.org/diffusion/EI/eimap.git", { tag, "0.4.0" } } }
+-   ]
+-}.
+-
+ { sub_dirs, [ "apps/egara" ] }.
+ { cover_enabled, true }.
+-{ relx, [ { release, { egara, "0.1" }, [egara]},
++{ relx, [ { release, { egara, "0.2" }, [egara]},
+           { dev_mode, false },
+           { include_erts, false },
+           { extended_start_script, true },
+Only in egara-0.2: rebar.lock

egara-0.2.tar.gz/.erlang Deleted

egara-0.2.tar.gz/.gitignore Changed

egara-0.2.tar.gz/Makefile Changed

egara-0.2.tar.gz/app.config Changed

@@ -3,7 +3,8 @@
 [
 { egara,
   [
-    { archival, true },
+    %% valid values: mail, groupware_object
+    { archival, [] },
 
     %% { worker_pool_size, 8 }, %% a sensible default is automatically generated if this is left undefined
 
@@ -41,8 +42,12 @@
               ]
     },
 
+    %% the type of node this is. recognized values:
+    %%    * receiver (gathers notifications from sources, e.g. imap)
+    %%    * sink (requires access to storage; also implies receiver)
+    { node_profile, receiver },
     %% the list of receivers on this node
-    { receivers, [ cyrus ] },
+    { receivers, [ cyrus_imap ] },
 
     %% only needs to be defined if using a non-standard socket and cyrus is in the receivers list
     { cyrus, [
@@ -96,13 +101,15 @@
 { lager,
   [
     {
-        handlers,
-        [
-            { lager_console_backend, info },
-            { lager_file_backend, [ { file, "log/error.log"}, { level, error } ] },
-            { lager_file_backend, [ { file, "log/console.log"}, { level, info } ] }
-        ] }
-  ] },
+     handlers,
+     [
+      { lager_console_backend, debug },
+      { lager_file_backend, [ { file, "log/error.log"}, { level, error } ] },
+      { lager_file_backend, [ { file, "log/console.log"}, { level, debug } ] }
+     ]
+    }
+  ]
+},
 { mnesia,
     [
      %%{dir, "db"}

egara-0.2.tar.gz/apps/egara/src/egara.app.src Changed

egara-0.2.tar.gz/apps/egara/src/egara.erl Changed

@@ -16,22 +16,29 @@
 %% along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 -module(egara).
-
 -behaviour(application).
 
 %% Application callbacks
--export([start/0, start/2, stop/1]).
+-export([start/0, start/2, stop/0, stop/1]).
 
-start() -> application:start(egara).
+start() -> ssl:start(), application:start(egara).
+stop() -> application:stop(egara).
 
 %% ===================================================================
 %% Application callbacks
 %% ===================================================================
 
 start(_StartType, _StartArgs) ->
+    %% TODO: automagic cluster setup
+    pg2:create(notifications_receivers),
+    syn:init(),
     lager:info("Starting Egara ..."),
     lager:info("    Initializing the mnesia-based notification queue ..."),
-    egara_notification_queue:install([node() | nodes()]),
+    % this should go back to a cluster once there is support for clusterd sinks
+    % otherwise, each receiver should keep local copies of its notifications that
+    % they bleed into the pool of sinks
+    % egara_notification_queue:install(node() | nodes()]),
+    egara_notification_queue:install([node()]),
     egara_notification_queue:start(),
     lager:info("    Clearing orphans since last start"),
     egara_notification_queue:release_all(),
@@ -40,4 +47,4 @@
     egara_sup:start_link().
 
 stop(_State) ->
-    ok.
+    application:stop(egara).

egara-0.2.tar.gz/apps/egara/src/egara_imap.erl Deleted

@@ -1,264 +0,0 @@
-%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
-%%
-%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
-%%
-%% This program is free software: you can redistribute it and/or modify
-%% it under the terms of the GNU General Public License as published by
-%% the Free Software Foundation, either version 3 of the License, or
-%% (at your option) any later version.
-%%
-%% This program is distributed in the hope that it will be useful,
-%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-%% GNU General Public License for more details.
-%%
-%% You should have received a copy of the GNU General Public License
-%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
--module(egara_imap).
-
--behaviour(gen_fsm).
-
-%% API
--export([start_link/0, start_link/1, connect/1, disconnect/1, get_folder_annotations/4, get_message_headers_and_body/5, get_path_tokens/3]).
-
-%% gen_fsm callbacks
--export([disconnected/2, authenticate/2, authenticating/2, idle/2, wait_response/2]).
--export([init/1, handle_event/3, handle_sync_event/4, handle_info/3, terminate/3, code_change/4]).
-
-%% state record definition
--record(state, { host, port, tls, user, pass, authed = false, socket,
-                 command_serial = 1, command_queue = queue:new(),
-                 current_command, current_mbox, parse_state }).
--record(command, { tag, mbox, message, from, response_token, parse_fun }).
-
-%% public API
-start_link() -> gen_fsm:start_link(?MODULE, [], []).
-start_link(_Args) -> gen_fsm:start_link(?MODULE, [], []).
-
-connect(PID) -> gen_fsm:send_all_state_event(PID, connect).
-disconnect(PID) -> gen_fsm:send_all_state_event(PID, disconnect).
-
-get_folder_annotations(PID, From, ResponseToken, Folder) when is_list(Folder) ->
-    get_folder_annotations(PID, From, ResponseToken, list_to_binary(Folder));
-get_folder_annotations(PID, From, ResponseToken, Folder) when is_binary(Folder) ->
-    Command = #command{ message = egara_imap_command_annotation:new(Folder),
-                        from = From, response_token = ResponseToken,
-                        parse_fun = fun egara_imap_command_annotation:parse/2 },
-    gen_fsm:send_all_state_event(PID, { ready_command, Command }).
-
-get_message_headers_and_body(PID, From, ResponseToken, Folder, MessageID) ->
-    %%lager:info("SELECT_DEBUG: peeking message ~p ~p", [Folder, MessageID]),
-    Command = #command{ mbox = Folder, message = egara_imap_command_peek_message:new(MessageID),
-                        from = From, response_token = ResponseToken,
-                        parse_fun = fun egara_imap_command_peek_message:parse/2 },
-    gen_fsm:send_all_state_event(PID, { ready_command, Command }).
-
-get_path_tokens(PID, From, ResponseToken) ->
-    Command = #command{ message = egara_imap_command_namespace:new(),
-                        from = From, response_token = ResponseToken,
-                        parse_fun = fun egara_imap_command_namespace:parse/2 },
-    gen_fsm:send_all_state_event(PID, { ready_command, Command }).
-
-%% gen_server API
-init(_Args) ->
-    Config = application:get_env(egara, imap, []),
-    AdminConnConfig = proplists:get_value(admin_connection, Config, []),
-    State = #state {
-                host = proplists:get_value(host, AdminConnConfig, ""),
-                port = proplists:get_value(port, AdminConnConfig, 993),
-                tls  = proplists:get_value(tls, AdminConnConfig, true),
-                user = list_to_binary(proplists:get_value(user, AdminConnConfig, "cyrus-admin")),
-                pass = list_to_binary(proplists:get_value(pass, AdminConnConfig, ""))
-              },
-    { ok, disconnected, State }.
-
-disconnected(connect, #state{ host = Host, port = Port, tls = TLS, socket = undefined } = State) ->
-    {ok, Socket} = create_socket(Host, Port, TLS),
-    { next_state, authenticate, State#state { socket = Socket } };
-disconnected(Command, State) when is_record(Command, command) ->
-    { next_state, disconnected, enque_command(Command, State) }.
-
-authenticate({ data, _Data }, #state{ user = User, pass = Pass, authed = false } = State) ->
-    Message = <<"LOGIN ", User/binary, " ", Pass/binary>>,
-    Command = #command{ message = Message },
-    send_command(Command, State),
-    { next_state, authenticating, State#state{ authed = in_process } };
-authenticate(Command, State) when is_record(Command, command) ->
-    { next_state, authenticate, enque_command(Command, State) }.
-
-authenticating({ data, Data }, #state{ authed = in_process } = State) ->
-    Token = <<" OK ">>, %TODO would be nice to have the tag there
-    case binary:match(Data, Token) of
-        nomatch ->
-            lager:warning("Log in to IMAP server failed: ~p", [Data]),
-            close_socket(State),
-            { next_state, idle, State#state{ socket = undefined, authed = false } };
-        _ ->
-            %%lager:info("Logged in to IMAP server successfully"),
-            gen_fsm:send_event(self(), process_command_queue),
-            { next_state, idle, State#state{ authed = true } }
-    end;
-authenticating(Command, State) when is_record(Command, command) ->
-    { next_state, authenticating, enque_command(Command, State) }.
-
-idle(process_command_queue, #state{ command_queue = Queue } = State) ->
-    case queue:out(Queue) of
-       { { value, Command }, ModifiedQueue } when is_record(Command, command) ->
-            %%lager:info("Clearing queue of ~p", [Command]),
-            StateWithNewQueue = State#state{ command_queue = ModifiedQueue },
-            NewState = send_command(Command, StateWithNewQueue),
-            { next_state, wait_response, NewState };
-       { empty, ModifiedQueue } ->
-            { next_state, idle, State#state{ command_queue = ModifiedQueue } }
-    end;
-idle({ data, Data }, State) ->
-    %%lager:info("Idling, server sent: ~p", [Data]),
-    { next_state, idle, State };
-idle(Command, State) when is_record(Command, command) ->
-    %%lager:info("Idling"),
-    NewState = send_command(Command, State),
-    { next_state, wait_response, NewState };
-idle(_Event, State) ->
-    { next_state, idle, State }.
-
-%%TODO a variant that checks "#command{ from = undefined }" to avoid parsing responses which will go undelivered?
-wait_response(Command, State) when is_record(Command, command) ->
-    { next_state, wait_response, enque_command(Command, State) };
-wait_response({ data, _Data }, #state{ current_command = #command{ parse_fun = undefined } } = State) ->
-    gen_fsm:send_event(self(), process_command_queue),
-    { next_state, idle, State };
-wait_response({ data, Data }, #state{ current_command = #command{ parse_fun = Fun, tag = Tag } } = State) when is_function(Fun, 2) ->
-    Response = Fun(Data, Tag),
-    %%lager:info("Response from parser was ~p ~p, size of queue ~p", [More, Response, queue:len(State#state.command_queue)]),
-    next_command_after_response(Response, State);
-wait_response({ data, Data }, #state{ parse_state = ParseState, current_command = #command{ parse_fun = Fun, tag = Tag } } = State) when is_function(Fun, 3) ->
-    Response = Fun(Data, Tag, ParseState),
-    %%lager:info("Response from parser was ~p ~p, size of queue ~p", [More, Response, queue:len(State#state.command_queue)]),
-    next_command_after_response(Response, State).
-
-handle_event(connect, disconnected, State) ->
-    gen_fsm:send_event(self(), connect),
-    { next_state, disconnected, State };
-handle_event(connect, _Statename, State) ->
-    %%lager:info("Already connected to IMAP server!"),
-    { next_state, _Statename, State };
-handle_event(disconnect, _StateName, State) ->
-    close_socket(State),
-    { next_state, disconnected, reset_state(State) };
-handle_event({ ready_command, Command }, StateName, State) when is_record(Command, command) ->
-    %%lager:info("Making command .. ~p", [Command]),
-    ?MODULE:StateName(Command, State);
-handle_event(_Event, StateName, State) -> { next_state, StateName, State}.
-
-handle_sync_event(_Event, _From, StateName, State) -> { next_state, StateName, State}.
-
-handle_info({ ssl, Socket, Bin }, StateName, #state{ socket = Socket } = State) ->
-    % Flow control: enable forwarding of next TCP message
-    %lager:info("Received from server: ~p", [Bin]),
-    ssl:setopts(Socket, [{ active, once }]),
-    ?MODULE:StateName({ data, Bin }, State);
-handle_info({ tcp, Socket, Bin }, StateName, #state{ socket = Socket } = State) ->
-    % Flow control: enable forwarding of next TCP message
-    %%lager:info("Got us ~p", [Bin]),
-    inet:setopts(Socket, [{ active, once }]),
-    ?MODULE:StateName({ data, Bin }, State);
-handle_info({tcp_closed, Socket}, _StateName, #state{ socket = Socket, host = Host } = State) ->
-    lager:info("~p Client ~p disconnected.\n", [self(), Host]),
-    { stop, normal, State };
-handle_info({ { selected, MBox }, ok }, StateName, State) ->
-    %%lager:info("~p Selected mbox ~p", [self(), MBox]),
-    { next_state, StateName, State#state{ current_mbox = MBox } };
-handle_info({ { selected, MBox }, { error, Reason } }, StateName, State) ->
-    lager:info("Failed to select mbox ~p: ~p", [MBox, Reason]),
-    NewQueue = queue:filter(fun(Command) -> notify_of_mbox_failure_during_filter(Command, Command#command.mbox =:= MBox) end, State#state.command_queue),
-    { next_state, StateName, State#state{ command_queue = NewQueue } };
-handle_info(_Info, StateName, State) ->
-    { next_state, StateName, State }.
-
-terminate(_Reason, _Statename, State) -> close_socket(State), ok.
-
-code_change(_OldVsn, Statename, State, _Extra) -> { ok, Statename, State }.
-
-%% private API
-notify_of_response(none, _Command) -> ok;
-notify_of_response(_Response, #command { from = undefined }) -> ok;
-notify_of_response(Response, #command { from = From, response_token = undefined }) -> From ! Response;
-notify_of_response(Response, #command { from = From, response_token = Token }) -> From ! { Token, Response };
-notify_of_response(_, _) -> ok.
-
-%% the return is inverted for filtering
-notify_of_mbox_failure_during_filter(Command, true) -> notify_of_response({ error, mailboxnotfound }, Command), false;
-notify_of_mbox_failure_during_filter(_Command, false) -> true.
-
-next_command_after_response({ more, Fun, ParseState }, State) when is_function(Fun, 3) ->
-    { next_state, wait_response, State#state{ parse_state = ParseState, current_command = State#state.current_command#command{ parse_fun = Fun } } };
-next_command_after_response({ more, ParseState }, State) ->
-    { next_state, wait_response, State#state{ parse_state = ParseState } };
-next_command_after_response({ error, _ } = ErrorResponse, State) ->
-    notify_of_response(ErrorResponse, State#state.current_command),

egara-0.2.tar.gz/apps/egara/src/egara_ldap.erl Changed

@@ -45,7 +45,7 @@
     { ok, State }.
 
 handle_call({ fetch_userdata, UserLogin } , From, State) when is_list(UserLogin) ->
-    handle_call({ store_userdata, erlang:list_to_binary(UserLogin) }, From, State);
+    handle_call({ store_userdata, egara_utils:as_binary(UserLogin) }, From, State);
 handle_call({ fetch_userdata, UserLogin }, _From, State) ->
     %%TODO perhaps call with continuation to avoid create new state everytime?
     NewState = ensure_connected(State),
@@ -57,7 +57,7 @@
 handle_cast(_Msg, State) ->
     { noreply, State }.
 
-handle_info({'EXIT', _ParentPid, shutdown}, State) ->
+handle_info({'EXIT', _ParentPid, shutdown}, _State) ->
     exit(shutdown);
 handle_info({'EXIT', From, _Reason}, State) ->
     %% look out for our ldap connection dropping
@@ -135,9 +135,9 @@
     LDAPAttributes = Entry#eldap_entry.attributes,
     %% TODO should it really be exporting ldap-specifics like "dn" and "cn"?
     [
-      { <<"id">>, list_to_binary(hd(proplists:get_value("nsuniqueid", LDAPAttributes, ""))) },
-      { <<"cn">>, list_to_binary(hd(proplists:get_value("cn", LDAPAttributes, ""))) },
-      { <<"dn">>, list_to_binary(Entry#eldap_entry.object_name) }
+      { <<"id">>, egara_utils:as_binary(hd(proplists:get_value("nsuniqueid", LDAPAttributes, ""))) },
+      { <<"cn">>, egara_utils:as_binary(hd(proplists:get_value("cn", LDAPAttributes, ""))) },
+      { <<"dn">>, egara_utils:as_binary(Entry#eldap_entry.object_name) }
     ];
 create_userdata_query_response(UserLogin, { error, Reason }) ->
     lager:warning("LDAP: Could not find requested user ~p, reason: ~p", [UserLogin, Reason]),
@@ -146,7 +146,3 @@
     lager:warning("LDAP: unknown error on user query"),
     notfound.
 
-%% UTILS
-userlogin_to_current_userdata_key(UserLogin) when is_list(UserLogin) -> userlogin_to_current_userdata_key(erlang:list_to_binary(UserLogin));
-userlogin_to_current_userdata_key(UserLogin) when is_binary(UserLogin) -> <<UserLogin/binary, "::current">>.
-

egara-0.2.tar.gz/apps/egara/src/egara_notification_queue.erl Added

@@ -0,0 +1,202 @@
+%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
+%%
+%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
+%%
+%% This program is free software: you can redistribute it and/or modify
+%% it under the terms of the GNU General Public License as published by
+%% the Free Software Foundation, either version 3 of the License, or
+%% (at your option) any later version.
+%%
+%% This program is distributed in the hope that it will be useful,
+%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+%% GNU General Public License for more details.
+%%
+%% You should have received a copy of the GNU General Public License
+%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+-module(egara_notification_queue).
+-export([ install/1, start/0,
+          notification/1, next_unassigned/0,
+          add/3, remove/1, remove_all/0,
+          assign_one/1, assigned_to/1,
+          release/1, release/2, release_orphaned/0, release_all/0,
+          migrate_failures/0,
+          max_key/0, list/0]).
+-include_lib("stdlib/include/qlc.hrl").
+-record(egara_incoming_notification, { id, node = undefined, claimed_by = undefined, fails = 0, term = undefined }).
+-record(egara_incoming_notification_fails, { id, node, term }).
+-define(MAX_FAILURES, 5).
+
+install(Nodes) ->
+    rpc:multicall(Nodes, application, stop, [mnesia]),
+    %% TODO: tune mnesia for large batches of writes; see -> http://streamhacker.com/2008/12/10/how-to-eliminate-mnesia-overload-events/
+    mnesia:create_schema(Nodes),
+    rpc:multicall(Nodes, application, start, [mnesia]),
+    %% create index for claimed_by
+    try
+        mnesia:create_table(egara_incoming_notification,
+                            [{ attributes, record_info(fields, egara_incoming_notification) },
+                             { type, ordered_set },
+                             { disc_copies, Nodes }]),
+        mnesia:create_table(egara_incoming_notification_fails,
+                            [{ attributes, record_info(fields, egara_incoming_notification_fails) },
+                             { disc_copies, Nodes }])
+    of _ -> ok
+    catch
+        error:_ -> ok
+    end.
+
+start() ->
+    mnesia:wait_for_tables([egara_incoming_notification], 5000).
+
+add(Key, Node, Term) ->
+    %lager:debug("Adding notification to the queue. Key: ~p => Value: ~p", [Key, Term]),
+    F = fun() ->
+                Rec = #egara_incoming_notification{ id = Key, node = Node, term = Term },
+                mnesia:write(Rec)
+        end,
+    mnesia:activity(transaction, F).
+
+remove(Key) ->
+    F = fun() -> try mnesia:delete({ egara_incoming_notification, Key }) of
+                     _ -> ok
+                 catch
+                     error:Error -> {error, caught, Error}
+                 end
+                 end,
+    mnesia:activity(transaction, F).
+
+remove_all() ->
+    mnesia:clear_table(egara_incoming_notification).
+
+notification(Key) ->
+    F = fun() ->
+                case mnesia:read(egara_incoming_notification, Key) of
+                    [Record] -> Record;
+                    [] -> #egara_incoming_notification{ id = 0 }
+                end
+        end,
+    mnesia:activity(transaction, F).
+
+release(Key, ProcessKey) ->
+    %%TODO: make this more efficient by using qlc; currently scans whole table!
+    Pattern = #egara_incoming_notification{ _ = '_', claimed_by = ProcessKey, id = Key },
+    F = fun() ->
+                %% check if claimed_by is set and if so if the process is still running
+                case mnesia:match_object(Pattern) of
+                    [#egara_incoming_notification{ term = Term }] -> mnesia:write(#egara_incoming_notification{ id = Key, term = Term, claimed_by = undefined });
+                    [] -> { error, notfound}
+                end
+        end,
+    mnesia:activity(transaction, F).
+
+release(Key) ->
+    F = fun() ->
+                case mnesia:read(egara_incoming_notification, Key) of
+                    [#egara_incoming_notification{ term = Term }] -> mnesia:write(#egara_incoming_notification{ id = Key, term = Term, claimed_by = undefined }), ok;
+                    [] -> false
+                end
+        end,
+    mnesia:activity(transaction, F).
+
+release_orphaned() ->
+    F = fun() ->
+                QH = qlc:q([ Record || #egara_incoming_notification{ claimed_by = ProcessKey } = Record <- mnesia:table(egara_incoming_notification),
+                             ProcessKey =/= undefined, syn:find_by_key(ProcessKey) =:= undefined ]),
+                qlc:fold(fun(Record, N) ->
+                                 Failed = Record#egara_incoming_notification.fails + 1,
+                                 mnesia:write(Record#egara_incoming_notification{ claimed_by = undefined, fails = Failed }),
+                                 N + 1
+                         end,
+                         0, QH)
+        end,
+    mnesia:activity(transaction, F).
+
+migrate_failures() ->
+    F = fun() ->
+                Timestamp = egara_utils:current_timestamp(),
+                QH = qlc:q([ Record || #egara_incoming_notification{ fails = Fails } = Record <- mnesia:table(egara_incoming_notification), Fails >= ?MAX_FAILURES ]),
+                qlc:fold(fun(#egara_incoming_notification{ id = Key, node = Node, term = Term }, N) ->
+                                 IntBin = egara_utils:as_binary(N),
+                                 mnesia:write(#egara_incoming_notification_fails{ id = <<Timestamp/binary, "_", IntBin/binary>>,
+                                                                                  node = Node,
+                                                                                  term = Term }),
+                                 remove(Key),
+                                 N + 1 end,
+                         0, QH)
+        end,
+    mnesia:activity(transaction, F).
+
+release_all() ->
+    F = fun() ->
+                QH = qlc:q([ Rec|| Rec <- mnesia:table(egara_incoming_notification),
+                             Rec#egara_incoming_notification.claimed_by =/= undefined ]),
+                qlc:fold(fun(Rec, N) -> mnesia:write(Rec#egara_incoming_notification{ claimed_by = undefined }), N + 1 end, 0, QH)
+        end,
+    mnesia:activity(transaction, F).
+
+assigned_to(Key) ->
+    F = fun() ->
+                QH = qlc:q([ Rec || Rec <- mnesia:table(egara_incoming_notification),
+                             Rec#egara_incoming_notification.id =:= Key]),
+                QC = qlc:cursor(QH),
+                Answers = qlc:next_answers(QC, 1),
+                case Answers of
+                    [Record|_] -> Record#egara_incoming_notification.claimed_by;
+                    _ -> notfound
+                end
+        end,
+    mnesia:activity(transaction, F).
+
+assign_one(ProcessKey) ->
+    F = fun() ->
+                QH = qlc:q([ Rec || Rec <- mnesia:table(egara_incoming_notification),
+                             Rec#egara_incoming_notification.claimed_by =:= undefined, Rec#egara_incoming_notification.fails < ?MAX_FAILURES]),
+                QC = qlc:cursor(QH),
+                Answers = qlc:next_answers(QC, 1),
+                case Answers of
+                    [Record|_] ->
+                        mnesia:write(Record#egara_incoming_notification{ claimed_by = ProcessKey }),
+                        incoming_as_ext_tuple(Record);
+                    _ -> notfound
+                end
+        end,
+    mnesia:activity(transaction, F).
+
+list() ->
+    F = fun() ->
+                QH = qlc:q([ Rec || Rec <- mnesia:table(egara_incoming_notification) ]),
+                QC = qlc:cursor(QH),
+                qlc:next_answers(QC)
+        end,
+    mnesia:activity(transaction, F).
+
+incoming_as_ext_tuple(#egara_incoming_notification{ id = Id, node = undefined, term = Term }) -> { Id, node(), Term };
+incoming_as_ext_tuple(#egara_incoming_notification{ id = Id, node = Node, term = Term }) -> { Id, Node, Term }.
+
+next_unassigned([]) ->
+    none;
+next_unassigned([H|_]) ->
+    H#egara_incoming_notification.id;
+next_unassigned(_) ->
+    error.
+next_unassigned() ->
+    F = fun() ->
+                QH = qlc:q([ Rec || Rec <- mnesia:table(egara_incoming_notification),
+                             Rec#egara_incoming_notification.claimed_by =:= undefined ]),
+                QC = qlc:cursor(QH),
+                Answers = qlc:next_answers(QC, 1),
+                next_unassigned(Answers)
+        end,
+    mnesia:activity(transaction, F).
+
+max_key() ->
+    F = fun() ->
+                case mnesia:last(egara_incoming_notification) of
+                    '$end_of_table' -> 0;
+                    Key -> Key
+                end

egara-0.2.tar.gz/apps/egara/src/egara_notifications_processor.erl Changed

@@ -20,7 +20,6 @@
 
 %% API
 -export([ start_link/0,
-          notifications_received/0,
           process_backlog/0,
           queue_drained/0
         ]).
@@ -31,9 +30,8 @@
 -define(BACKLOG_BATCHSIZE, 1).
 
 %% API
-start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-notifications_received() -> gen_server:cast(?MODULE, { notifications_received }).
-process_backlog() -> gen_server:cast(?MODULE, { process_backlog }).
+start_link() -> gen_server:start_link({ global, ?MODULE }, ?MODULE, [], []).
+process_backlog() -> gen_server:cast({ global, ?MODULE }, { process_backlog }).
 queue_drained() -> process_backlog().
 
 %% private/internal functions
@@ -41,38 +39,39 @@
     %%lager:info("ERRRORS, AND NO NOTIFICATIONS!"),
     ok;
 process_backlog_if_notifications(none) ->
-    egara_notification_queue:migrate_failures(),
-    process_backlog_if_notifications(egara_notification_queue:release_orphaned()),
     %%lager:info("NO NOTIFICATIONS! ~p", [egara_notification_queue:next_unassigned()]),
-    ok;
-process_backlog_if_notifications(0) ->
-    %%lager:info("ZERO NOTIFICATIONS!"),
-    ok;
+    egara_notification_queue:migrate_failures(),
+    case egara_notification_queue:release_orphaned() of
+        0 -> ok;
+        _ -> timer:apply_after(5000, ?MODULE, process_backlog, []), ok
+    end;
 process_backlog_if_notifications(N) when is_number(N) ->
-    case start_a_worker(0) of
-        0 -> timer:apply_after(500, ?MODULE, process_backlog, []), ok;
+    case start_a_worker() of
+        done -> timer:apply_after(500, ?MODULE, process_backlog, []), ok;
         _ -> ok
     end;
 process_backlog_if_notifications(_) ->
     %%lager:info("GOT SOMETHING WE DIDN'T EXPECT?"),
     ok.
+
 process_backlog_if_notifications() ->
     %%lager:info("Max key: ~p, assigned to: ~p", [egara_notification_queue:max_key(), egara_notification_queue:assigned_to(egara_notification_queue:max_key())]),
     process_backlog_if_notifications(egara_notification_queue:next_unassigned()).
 
-start_a_worker(N) ->
+start_a_worker() ->
     try
         case poolboy:checkout(egara_notification_workers, false, 10) of
             Worker when is_pid(Worker) ->
                 %%lager:info("Checked out ~p", [Worker]),
                 gen_server:cast(Worker, process_events),
-                start_a_worker(N + 1);
+                process_backlog(),
+                ok;
             _ ->
-                N
+                done
         end
     catch
         _ ->
-            N
+            done
     end.
 
 
@@ -85,11 +84,8 @@
 handle_call(_, _From, State) ->
     { reply, ok, State }.
 
-handle_cast({ notifications_received }, State) ->
-    { noreply, State };
-
 handle_cast({ process_backlog }, State) ->
-    lager:info("Handling backlog...."),
+    %%lager:info("Handling backlog...."),
     process_backlog_if_notifications(),
     { noreply, State };

egara-0.2.tar.gz/apps/egara/src/egara_notifications_receiver.erl Changed

@@ -31,8 +31,8 @@
 -define(NOTIFICATION_BATCH_SIZE, 500).
 
 %% API
-start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-notification_received(NotificationJson) -> gen_server:cast(?MODULE, { notification, NotificationJson }).
+start_link() -> gen_server:start_link(?MODULE, [], []).
+notification_received(NotificationJson) -> gen_server:cast(pg2:get_closest_pid(notifications_receivers), { notification_received, node(), NotificationJson }).
 
 %% private/internal functions
 inform_notifications_processor(N) when is_number(N), N >= ?NOTIFICATION_BATCH_SIZE ->
@@ -41,10 +41,11 @@
 inform_notifications_processor(N) ->
     N.
 
-%%TODO this process could be started only if needed, killed when not?
 notifications_processor_notifier(Total) when is_number(Total) ->
     receive
-        Pending when is_number(Pending) -> notifications_processor_notifier(inform_notifications_processor(Total + Pending))
+        Pending when is_number(Pending) ->
+            NewTotal = inform_notifications_processor(Total + Pending),
+            notifications_processor_notifier(NewTotal)
     after 1000 ->
         if Total > 0 -> inform_notifications_processor(?NOTIFICATION_BATCH_SIZE);
            true -> ok
@@ -52,29 +53,17 @@
         notifications_processor_notifier(0)
     end.
 
-start_receiver([]) -> ok;
-start_receiver([cyrus|Tail]) -> lager:info("Starting receiver: cyrus"), egara_incoming_cyrus_imap:start_reception(), start_receiver(Tail);
-start_receiver([_|Tail]) -> start_receiver(Tail).
-
-start_notification_reception() ->
-    case application:get_env(receivers) of
-        { ok, Receivers } when is_list(Receivers) -> start_receiver(Receivers);
-        _ -> ok
-    end.
-
-%% gen_server callbacks
 init([]) ->
+    pg2:join(notifications_receivers, self()),
     MaxKey = egara_notification_queue:max_key(),
-    Rv = start_notification_reception(),
-    %%TODO: on Rv = error, do something appropriate
-    lager:info("Notification reception started ... ~p", [Rv]),
+    lager:info("Notification reception started ... "),
     { ok, #state{ storage_id = MaxKey + 1, processor_notifier_pid = spawn(fun() -> notifications_processor_notifier(0) end) } }.
 
 handle_call(_, _From, State) ->
     { reply, ok, State }.
 
-handle_cast({ notification, Notification }, State) ->
-    egara_notification_queue:add(State#state.storage_id, Notification),
+handle_cast({ notification_received, SourceNode, Notification }, State) ->
+    egara_notification_queue:add(State#state.storage_id, SourceNode, Notification),
     State#state.processor_notifier_pid ! 1,
     { noreply, State#state{ storage_id = State#state.storage_id + 1 } }; %% if paralellized, this needs to be syncronized

egara-0.2.tar.gz/apps/egara/src/egara_riak_config.erl Changed

@@ -68,17 +68,16 @@
     end;
 sanitize_node_config(missing) -> [ { "localhost", 10017 } ].
 
-next_node(State) when is_record(State, state) -> next_node(length(State#state.nodes), State).
-next_node(Length, #state{ pos = Pos, nodes = Nodes } = State) when Pos > length(State#state.nodes) ->
+next_node(#state{ pos = Pos, nodes = Nodes } = State) when Pos > length(State#state.nodes) ->
     [ Tuple | _ ] = Nodes,
     { reply, Tuple, State#state{ pos = 2 } };
-next_node(_Length, #state{ pos = Pos, nodes = Nodes } = State) ->
+next_node(#state{ pos = Pos, nodes = Nodes } = State) ->
     Tuple = lists:nth(Pos, Nodes),
     { reply, Tuple, State#state{ pos = Pos + 1 } }.
 
 %%TODO: current almost entirely useless since the API does not allow creating new bucket types
 setup_bucket_types(Connection, [{ BucketList, Properties } | Tail]) -> 
-    Bucket = list_to_binary(BucketList),
+    Bucket = egara_utils:as_binary(BucketList),
     CurrentTypeProperties = riakc_pb_socket:get_bucket_type(Connection, Bucket),
     case CurrentTypeProperties of
         { ok, _ } -> %% TODO check that the properties actually match up so as not to set the type up EVERY time

egara-0.2.tar.gz/apps/egara/src/egara_storage.erl Changed

@@ -51,18 +51,18 @@
     { ok, #state {} }.
 
 handle_call({ store_notification, Keys, Notification }, _From, State) when is_list(Keys) ->
-    %%lager:info("Notification----> ~p = ~p", [Keys, Notification]),
+    lager:info("Notification----> ~p = ~p", [Keys, Notification]),
     NewState = ensure_connected(State),
     Json = jsx:encode(Notification),
     store_notification_json(Keys, Json, NewState, {});
 handle_call({ store_notification, Key, Notification }, _From, State) when is_binary(Key) ->
-    %%lager:info("Notification----> ~p = ~p", [Key, Notification]),
+    lager:info("Notification----> ~p = ~p", [Key, Notification]),
     NewState = ensure_connected(State),
     Json = jsx:encode(Notification),
     store_notification_json(Key, Json, NewState);
 
 handle_call({ store_userdata, UserLogin, UserData }, From, State) when is_list(UserLogin) ->
-    handle_call({ store_userdata, erlang:list_to_binary(UserLogin), UserData }, From, State);
+    handle_call({ store_userdata, egara_utils:as_binary(UserLogin), UserData }, From, State);
 handle_call({ store_userdata, UserLogin, UserData }, _From, State) ->
     UserId = proplists:get_value(<<"id">>, UserData, ""),
     TS = egara_utils:current_timestamp(),
@@ -81,7 +81,7 @@
     end;
 
 handle_call({ fetch_userdata, UserLogin }, From, State) when is_list(UserLogin) ->
-    handle_call({ store_userdata, erlang:list_to_binary(UserLogin) }, From, State);
+    handle_call({ store_userdata, egara_utils:as_binary(UserLogin) }, From, State);
 handle_call({ fetch_userdata, UserLogin }, _From, State) when is_binary(UserLogin) ->
     NewState = ensure_connected(State),
     RiakResponse = riakc_pb_socket:get(NewState#state.riak_connection, current_users_bucket(), UserLogin),
@@ -105,7 +105,7 @@
     { reply, Rv, NewState };
 
 handle_call({ fetch_folder_uid, Folder }, From, State) when is_list(Folder) ->
-    handle_call({ fetch_folder_uid, list_to_binary(Folder) }, From, State);
+    handle_call({ fetch_folder_uid, egara_utils:as_binary(Folder) }, From, State);
 handle_call({ fetch_folder_uid, Folder }, _From, State) when is_binary(Folder) ->
     NewState = ensure_connected(State),
     RiakResponse = riakc_pb_socket:get(NewState#state.riak_connection, current_folders_bucket(), Folder),
@@ -179,4 +179,4 @@
 %%historical_folders_bucket() -> { <<"egara-lww">>, <<"imap-folders">> }.
 current_folders_bucket() -> { <<"egara-unique">>, <<"imap-folders-current">> }.
 message_timeline_bucket() -> { <<"egara-lww">>, <<"imap-message-timeline">> }.
-json_type() -> <<"application/json">>.
+json_type() -> "application/json".

egara-0.2.tar.gz/apps/egara/src/egara_sup.erl Changed

@@ -27,6 +27,7 @@
 
 %% Helper macro for declaring children of supervisor
 -define(CHILD(Module, Type), { Module, { Module, start_link, []}, permanent, 5000, Type, [Module] }).
+-define(RECEIVER_PREFIX, "egara_incoming_").
 
 %% ===================================================================
 %% API functions
@@ -41,22 +42,44 @@
 
 init([]) ->
     lager:info("    Creating resource pools ..."),
-    WorkerSize = application:get_env(egara, worker_pool_size, erlang:system_info(schedulers_online) * 2),
     %% there is one more egara_imap process to allow non-egara_worker process access as needed through a shared one
-    Pools = [
-             { egara_ldap_pool, egara_ldap, [ { size, WorkerSize }, { max_overflow, 0 }], [ ] },
-             { egara_imap_pool, egara_imap, [ { size, WorkerSize + 1 }, { max_overflow, 0 }], [ ] },
-             { egara_notification_workers, egara_worker, [ { size, WorkerSize }, { max_overflow, 0 }], [ ] }
-            ],
+    NodeType = application:get_env(egara, node_profile, sink),
+    io:format("This is an Egara node of type ~p~n", [NodeType]),
+    Pools = worker_pool_specs(NodeType, []),
     PoolSpecs = lists:map(fun({Name, Module, PoolConfig, WorkerArgs}) ->
                                   PoolArgs = [{ name, { local, Name } }, { worker_module, Module }],
                                   poolboy:child_spec(Name, PoolArgs ++ PoolConfig, WorkerArgs)
                           end, Pools),
     %lager:info("Pools: ~p", [PoolSpecs]),
-    Children = [
-                 ?CHILD(egara_riak_config, worker),
-                 ?CHILD(egara_notifications_receiver, worker),
-                 ?CHILD(egara_notifications_processor, worker)
-               ],
+    Children = notification_processes(NodeType, []),
     { ok, { { one_for_one, 5, 10}, PoolSpecs ++ Children } }.
 
+worker_pool_specs(sink, Acc) ->
+    WorkerSize = application:get_env(egara, worker_pool_size, erlang:system_info(schedulers_online) * 2),
+    SinkPools =
+    [{ egara_notification_workers, egara_worker, [ { size, WorkerSize }, { max_overflow, 0 }], [ ] },
+     { egara_ldap_pool, egara_ldap, [ { size, WorkerSize }, { max_overflow, 0 }], [ ] }
+     | Acc],
+    worker_pool_specs(receiver, SinkPools);
+worker_pool_specs(receiver, Acc) -> Acc.
+
+notification_processes(sink, Acc) ->
+    Children =
+    [?CHILD(egara_riak_config, worker),
+     ?CHILD(egara_notifications_processor, worker),
+     ?CHILD(egara_notifications_receiver, worker) | Acc],
+    notification_processes(receiver, Children);
+notification_processes(receiver, Acc) ->
+    spawn(fun() -> start_receivers() end),
+    Acc.
+
+start_receivers() ->
+    timer:sleep(1000),
+    start_receiver(application:get_env(egara, receivers, [])).
+
+start_receiver([]) -> ok;
+start_receiver([Module|Tail]) when is_atom(Module) -> Atom = list_to_atom(?RECEIVER_PREFIX ++ atom_to_list(Module)), Atom:start_reception(), start_receiver(Tail);
+start_receiver([Module|Tail]) when is_list(Module) -> Atom = list_to_atom(?RECEIVER_PREFIX ++ Module), Atom:start_reception(), start_receiver(Tail);
+start_receiver([_Module|Tail]) -> start_receiver(Tail);
+start_receiver(_NotAList) -> ok.
+

egara-0.2.tar.gz/apps/egara/src/egara_utils.erl Added

@@ -0,0 +1,27 @@
+%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
+%%
+%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
+%%
+%% This program is free software: you can redistribute it and/or modify
+%% it under the terms of the GNU General Public License as published by
+%% the Free Software Foundation, either version 3 of the License, or
+%% (at your option) any later version.
+%%
+%% This program is distributed in the hope that it will be useful,
+%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+%% GNU General Public License for more details.
+%%
+%% You should have received a copy of the GNU General Public License
+%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+-module(egara_utils).
+-export([ current_timestamp/0, normalize_timestamp/1, as_binary/1 ]).
+
+current_timestamp() -> iso8601:format(os:timestamp()).
+normalize_timestamp(Timestamp) -> iso8601:format(iso8601:parse_exact(Timestamp)).
+
+as_binary(Value) when is_binary(Value) -> Value;
+as_binary(Value) when is_list(Value) -> erlang:list_to_binary(Value);
+as_binary(Value) when is_integer(Value) -> erlang:integer_to_binary(Value);
+as_binary(Value) when is_float(Value) -> erlang:float_to_binary(Value).

egara-0.2.tar.gz/apps/egara/src/egara_worker.erl Changed

@@ -26,24 +26,26 @@
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
 -define(BATCH_SIZE, 500).
 
--record(state, { archival = true, event_mapping, storage, imap, user_blacklist, imap_path_delim = "/", imap_shared_prefix = none }).
--record(message_peek_data, { timestamp, folder_path, folder_uid, notification, notification_queue_key, message_uid, uid_set, old_folder_uid, old_message_uid, old_uid_set }).
--record(message_event_getfolderuid_data, { folder, notification_queue_key, notification, event_type }).
--record(message_event_getoldfolderuid_data, { folder, folder_uid, uidset, notification_queue_key, notification, old_folder_path, old_uid_set }).
+-record(state, { archival = [], event_mapping, storage, imap, user_blacklist, imap_path_delim = "/", imap_shared_prefix = none, process_key }).
+-record(message_event_getfolderuid_data, { folder, notification_queue_key, source_node, notification, event_type }).
+-record(message_event_getoldfolderuid_data, { folder_uid, uidset, source_node, notification_queue_key, notification, old_folder_path, old_uid_set }).
 
 start_link(Args) -> gen_server:start_link(?MODULE, Args, []).
 
 init(_Args) ->
     %% EventMapping is a map of event types (e.g. <<"FlagsClear">>) to the type of
     %% event (e.g. imap_mesage_event) for routing the notifications through the handler
-    EventMapping = transform_events_config_to_dict(application:get_env(events_to_track)),
+    EventMapping = transform_events_config_to_dict(application:get_env(egara, events_to_track)),
     { ok, Storage } = egara_storage:start_link(),
     %% get the admin user, whose events we will ignore
-    Archival = application:get_env(egara, archival, false),
+    Archival = application:get_env(egara, archival, []),
     ImapConfig = application:get_env(egara, imap, []),
     UserBlacklist = lists:foldl(fun(User, Acc) -> [list_to_binary(User)|Acc] end, [], proplists:get_value(user_blacklist, ImapConfig, [])),
-    Imap = start_imap(),
-    State = #state{ archival = Archival, event_mapping = EventMapping, imap = Imap, storage = Storage, user_blacklist = UserBlacklist },
+    ImapServerConfig = proplists:get_value(admin_connection, ImapConfig),
+    Imap = start_imap(ImapServerConfig),
+    ProcessKey = { node(),  erlang:unique_integer([monotonic]) },
+    syn:register(ProcessKey, self()),
+    State = #state{ archival = Archival, event_mapping = EventMapping, imap = Imap, storage = Storage, user_blacklist = UserBlacklist, process_key = ProcessKey },
     { ok, State }.
 
 handle_call(_Request, _From, State) ->
@@ -61,33 +63,32 @@
     NewState = State#state{ imap_shared_prefix = SharedPrefix, imap_path_delim = Delim },
     { noreply, NewState };
 handle_info({ { imap_mailbox_metadata, Folder, NotificationQueueKey, Notification }, Metadata }, State) ->
-    lager:info("Got imap_mailbox_metadata ~p", [Metadata]),
-    Uid = proplists:get_value(egara_imap_utils:header_name(mailbox_uid), Metadata, undefined),
+    %lager:info("Got imap_mailbox_metadata ~p", [Metadata]),
+    Uid = proplists:get_value(eimap_utils:header_name(mailbox_uid), Metadata, undefined),
     %%lager:info("Uid is ~p", [Uid]),
     NotificationWithMetadata = [ { <<"metadata">>, Metadata } | Notification ],
     Result = store_folder_notification_with_uid(Uid, Folder, NotificationWithMetadata, State#state.storage),
     notification_processing_result(NotificationQueueKey, Result, State),
     { noreply, State };
 handle_info({ #message_event_getfolderuid_data{ folder = Folder, notification_queue_key = NotificationQueueKey,
-                                                 notification = Notification, event_type = EventType }, Metadata }, State) ->
+                                                source_node = SourceNode, notification = Notification, event_type = EventType }, Metadata }, State) ->
     %%TODO: and if we somehow end up with a folder we can't find, or the uniqueid is not there?
-    FolderUid = proplists:get_value(<<"/vendor/cmu/cyrus-imapd/uniqueid">>, Metadata),
+    FolderUid = proplists:get_value(eimap_utils:header_name(mailbox_uid), Metadata),
     %%lager:info("Folder Uid to be stored ~p", [FolderUid]),
     egara_storage:store_folder_uid(State#state.storage, Folder, FolderUid),
-    Result = store_message_event(State, FolderUid, Notification, EventType, NotificationQueueKey),
+    Result = store_message_event(State, FolderUid, SourceNode, Notification, EventType, NotificationQueueKey),
     notification_processing_result(NotificationQueueKey, Result, State),
     { noreply, State };
-handle_info({ #message_event_getoldfolderuid_data{ folder = FolderPath, folder_uid = FolderUid, uidset = UidSet,
+handle_info({ #message_event_getoldfolderuid_data{ folder_uid = FolderUid, uidset = UidSet,
                                                    notification_queue_key = NotificationQueueKey,
+                                                   source_node = SourceNode,
                                                    notification = Notification,
                                                    old_folder_path = OldFolderPath, old_uid_set = OldUidSet}, Metadata }, State) ->
     OldFolderUid = proplists:get_value(<<"/vendor/cmu/cyrus-imapd/uniqueid">>, Metadata),
     egara_storage:store_folder_uid(State#state.storage, OldFolderPath, OldFolderUid),
     Timestamp = timestamp_from_notification(Notification),
-    start_message_peek(State#state.imap, Timestamp, Notification, FolderPath, FolderUid, UidSet, OldFolderUid, OldUidSet, NotificationQueueKey),
-    { noreply, State };
-handle_info({ MessagePeekData, Result }, State) when is_record(MessagePeekData, message_peek_data) ->
-    message_peek_received(State, MessagePeekData, Result),
+    Result = store_history_with_archival(State, SourceNode, Notification, Timestamp, FolderUid, UidSet, OldFolderUid, OldUidSet, NotificationQueueKey),
+    notification_processing_result(NotificationQueueKey, Result, State),
     { noreply, State };
 handle_info(_Info, State) ->
     { noreply, State }.
@@ -101,11 +102,15 @@
 %% private API
 
 %% init() helpers
-start_imap() ->
-    { ok, Imap } = egara_imap:start_link(),
-    egara_imap:connect(Imap),
-    egara_imap:get_path_tokens(Imap, self(), get_path_tokens),
-    Imap.
+start_imap(ImapServerConfig) ->
+    { ok, ImapSession } = eimap:start_link(ImapServerConfig),
+    eimap:connect(ImapSession),
+    %%eimap:starttls(ImapSession, undefined, undefined),
+    eimap:login(ImapSession, undefined, undefined,
+                proplists:get_value(user, ImapServerConfig, ""),
+                proplists:get_value(pass, ImapServerConfig, "")),
+    eimap:get_path_tokens(ImapSession, self(), get_path_tokens),
+    ImapSession.
 
 transform_events_config_to_dict({ ok, EventConfig }) ->
     lists:foldl(fun({ Type, Events }, EventMap) -> add_events_to_dict(Type, Events, EventMap) end, dict:new(), EventConfig);
@@ -124,50 +129,33 @@
 add_events_to_dict(_Type, _Events, EventMap) ->
     EventMap.
 
-%% 
-message_peek_received(State, #message_peek_data{ folder_path = FolderPath, message_uid = MessageUid, notification = Notification } = MessagePeekData, { error, Reason }) ->
-    lager:error("Message ~p in ~p not found; reason: ~p", [MessageUid, FolderPath, Reason]),
-    message_peek_iteration(MessagePeekData, Notification, State),
-    { noreply, State };
-message_peek_received(State, #message_peek_data{ notification = Notification } = MessagePeekData, Data) ->
-    PeekedNotification = lists:foldl(fun(Atom, Notif) -> add_entry_to_notification(Atom, Data, Notif) end, Notification, [flags, headers, message]),
-    message_peek_iteration(MessagePeekData, PeekedNotification, State),
-    { noreply, State }.
-
-message_peek_iteration(#message_peek_data{ timestamp = Timestamp, folder_path = FolderPath, folder_uid = FolderUid, notification = Notification,
-                                           notification_queue_key = NotificationQueueKey, message_uid = MessageUid, uid_set = UidSet,
-                                           old_folder_uid = OldFolderUid, old_message_uid = OldMessageUid, old_uid_set = OldUidSet },
-                         StorableNotification, #state { imap = Imap, storage = Storage } = State) ->
-    GroupwareUid = proplists:get_value(<<"groupware_uid">>, StorableNotification),
-    egara_history_entry:store(Storage, Timestamp, GroupwareUid, FolderUid, MessageUid, OldFolderUid, OldMessageUid),
-    Key = generate_message_event_key(FolderUid, MessageUid, Timestamp),
-    egara_storage:store_notification(Storage, Key, StorableNotification),
-    case start_message_peek(Imap, Timestamp, Notification,
-                            FolderPath, FolderUid, UidSet,
-                            OldFolderUid, OldUidSet,
-                            NotificationQueueKey) of
-        done ->
-            notification_processing_result(NotificationQueueKey, ok, State);
-        _ -> ok
-    end.
-
-add_entry_to_notification(headers, Data, Notification) ->
-    Headers = proplists:get_value(headers, Data),
+%%
+add_entry_to_notification(Key, undefined, Notification) when is_binary(Key) ->
+    Notification;
+add_entry_to_notification(headers, Value, Notification) when is_list(Value) ->
     NotificationWithGroupwareUid =
-    case groupware_uid_from_headers(Headers) of
+    case groupware_uid_from_headers(Value) of
         undefined -> Notification;
         GroupwareUid -> [{ <<"groupware_uid">>, GroupwareUid } | Notification]
     end,
-    add_entry_to_notification(<<"headers">>, Headers, NotificationWithGroupwareUid);
-add_entry_to_notification(Atom, Data, Notification) when is_atom(Atom) ->
+    [{ <<"headers">>, Value } | NotificationWithGroupwareUid];
+add_entry_to_notification(Atom, Value, Notification) when is_atom(Atom) ->
     Key = atom_to_binary(Atom, utf8),
-    Value = proplists:get_value(Atom, Data),
-    add_entry_to_notification(Key, Value, Notification);
-add_entry_to_notification(Key, undefined, Notification) when is_binary(Key) ->
-    Notification;
+    [{ Key, Value } | Notification];
 add_entry_to_notification(Key, Value, Notification) when is_binary(Key) ->
     [{ Key, Value } | Notification].
 
+groupware_uid_from_headers(Headers) when is_list(Headers) ->
+    groupware_uid_from_headers(Headers, proplists:get_value(<<"X-Kolab-Type">>, Headers));
+groupware_uid_from_headers(_) ->
+    undefined.
+
+groupware_uid_from_headers(_Headers, undefined) ->
+    undefined;
+groupware_uid_from_headers(Headers, _) ->
+    proplists:get_value(<<"Subject">>, Headers).
+
+
 store_folder_notification_with_uid(undefined, _Folder, Notification, _Storage) ->
     lager:warning("Could not find Uid for notification ~p", [Notification]),
     ok;
@@ -177,101 +165,90 @@
     egara_storage:store_folder_uid(Storage, Folder, Uid),
     egara_storage:store_notification(Storage, Key, Notification).
 
-generate_folder_event_key(Uid, Notification) ->
-    Timestamp = timestamp_from_notification(Notification),
-    <<"mailbox::", Uid/binary, "::", Timestamp/binary>>.
-
-store_message_event(State, FolderUid, Notification, EventType, NotificationQueueKey)
+store_message_event(State, FolderUid, SourceNode, Notification, EventType, NotificationQueueKey)
     when EventType =:= <<"MessageNew">> orelse
          EventType =:= <<"MessageAppend">> orelse
          EventType =:= <<"MessageCopy">> orelse
          EventType =:= <<"MessageMove">> ->
+    %lager:info("Storing a message event of type ~p", [EventType]),
     %% we need to fetch message content here and begin generating history events while doing so
     { NotificationWithUidSet, UidSet } = uidset_from_notification(Notification),
-    FolderPath = normalized_folder_path_from_notification(Notification, State),
     OldFolderUri = proplists:get_value(<<"oldMailboxID">>, Notification),
     case old_location(State, Notification, OldFolderUri) of
         { undefined, OldUidSet } ->
             %% we don't have the UID for the older folder, let's go fetch it
-            OldFolderPath = egara_imap_utils:extract_path_from_uri(State#state.imap_shared_prefix, State#state.imap_path_delim, OldFolderUri),
-            Data = #message_event_getoldfolderuid_data{ folder = FolderPath, folder_uid = FolderUid,
+            OldFolderPath = eimap_utils:extract_path_from_uri(State#state.imap_shared_prefix, State#state.imap_path_delim, OldFolderUri),
+            lager:debug("Fetching folder UID for ~p", [OldFolderPath]),
+            Data = #message_event_getoldfolderuid_data{ folder_uid = FolderUid,
                                                         uidset = UidSet,
                                                         notification_queue_key = NotificationQueueKey,
+                                                        source_node = SourceNode,
                                                         notification = NotificationWithUidSet,

egara-0.2.tar.gz/apps/egara/src/incoming_handlers/egara_incoming_cyrus_imap.erl Changed

egara-0.2.tar.gz/apps/egara/src/incoming_handlers/egara_payload_cyrus.erl Added

@@ -0,0 +1,135 @@
+%% Copyright 2014-2016 Kolab Systems AG (http://www.kolabsys.com)
+%%
+%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
+%%
+%% This program is free software: you can redistribute it and/or modify
+%% it under the terms of the GNU General Public License as published by
+%% the Free Software Foundation, either version 3 of the License, or
+%% (at your option) any later version.
+%%
+%% This program is distributed in the hope that it will be useful,
+%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+%% GNU General Public License for more details.
+%%
+%% You should have received a copy of the GNU General Public License
+%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+-module(egara_payload_cyrus).
+-export([remote_payload_for_message/5, async_payload_for_message/5, payload_for_message/4,
+         remote_groupwareuid_for_message/5, async_groupwareuid_for_message/5, groupwareuid_for_message/4]).
+
+remote_payload_for_message(Node, SharedPrefix, PathDelim, FolderUri, MessageUid) ->
+    spawn_link(Node, ?MODULE, async_payload_for_messages, [self(), SharedPrefix, PathDelim, FolderUri, MessageUid]),
+    %%TODO: error on timeout
+    receive
+        { payload_for_message, Response } -> Response
+    end.
+
+-spec async_payload_for_message(Pid :: pid(), SharedPrefix :: string(), PathDelim :: string(), FolderUri :: binary(), MessageUid :: integer()) -> list({ Key :: binary(), Value :: binary() }).
+async_payload_for_message(Pid, SharedPrefix, PathDelim, FolderUri, MessageUid) ->
+    Pid ! { payload_for_message, payload_for_message(SharedPrefix, PathDelim, FolderUri, MessageUid) }.
+
+-spec payload_for_message(SharedPrefix :: string(), PathDelim :: string(), FolderUri :: binary(), MessageUid :: integer()) -> list({ Key :: binary(), Value :: binary() }).
+payload_for_message(SharedPrefix, PathDelim, FolderUri, MessageUid) ->
+    %%TODO determine if this can / should be cached to avoid repetition
+    FsPath = filesystem_folder_path(SharedPrefix, PathDelim, FolderUri),
+    FsFilePath = FsPath ++ "/" ++ integer_to_list(MessageUid) ++ ".",
+    parse_payload(file:open(FsFilePath, [read, raw, binary])).
+
+-spec parse_payload({ ok, IoDevice :: any() } | any()) -> binary().
+parse_payload({ ok, IoDevice }) ->
+    Rv = parse_payload_headers(IoDevice),
+    file:close(IoDevice),
+    Rv;
+parse_payload(_) ->
+    %% could not open file, just return nothing
+    [ { flags, [] }, { headers, [] }, { message, <<"">> }].
+
+parse_payload_headers(IoDevice) ->
+    parse_payload_headers(IoDevice, [], [], <<>>, <<>>, file:read_line(IoDevice)).
+
+parse_payload_headers(IoDevice, Flags, Headers, Body, CurrentHeader, { ok, <<"\n">> = Line }) ->
+    parse_payload_body(IoDevice, Flags, add_header(CurrentHeader, Headers), <<Body/binary, Line/binary>>);
+parse_payload_headers(IoDevice, Flags, Headers, Body, CurrentHeader, { ok, <<"--=", _Value/binary>> = Line}) ->
+    parse_payload_body(IoDevice, Flags, add_header(CurrentHeader, Headers), <<Body/binary, Line/binary>>);
+parse_payload_headers(IoDevice, Flags, Headers, Body, CurrentHeader, { ok, <<" ", Rest/binary>> = Line }) ->
+    parse_payload_headers(IoDevice, Flags, Headers, <<Body/binary, Line/binary>>, <<CurrentHeader/binary, Rest/binary>>, file:read_line(IoDevice));
+parse_payload_headers(IoDevice, Flags, Headers, Body, CurrentHeader, { ok, Line }) ->
+    parse_payload_headers(IoDevice, Flags, add_header(CurrentHeader, Headers), <<Body/binary, Line/binary>>, Line, file:read_line(IoDevice));
+parse_payload_headers(_IoDevice, Flags, Headers, Body, CurrentHeader, _) ->
+    %% error or end of file
+    [ { flags, Flags }, { headers, add_header(CurrentHeader, Headers) }, { message, Body }].
+
+add_header(<<>>, Headers) ->
+    Headers;
+add_header(NewHeader, Headers) ->
+    case binary:match(NewHeader, <<": ">>) of
+        nomatch ->
+            Headers;
+        { Start, Length } ->
+            [{ binary:part(NewHeader, 0, Start), binary:part(NewHeader, Start + Length, size(NewHeader) - Start - Length - 1) } | Headers]
+    end.
+
+parse_payload_body(IoDevice, Flags, Headers, Body) ->
+    parse_payload_body(IoDevice, Flags, Headers, Body, file:read(IoDevice, 8 * 1024)).
+
+parse_payload_body(IoDevice, Flags, Headers, Body, { ok, Data }) ->
+    parse_payload_body(IoDevice, Flags, Headers, <<Body/binary, Data/binary>>, file:read(IoDevice, 8 * 1024));
+parse_payload_body(_IoDevice, Flags, Headers, Body, _) ->
+    [ { flags, Flags }, { headers, Headers }, { message, Body }].
+
+
+
+-spec remote_groupwareuid_for_message(Node :: node(), SharedPrefix :: string(), PathDelim :: string(), FolderUri :: binary(), MessageUid :: integer()) -> binary().
+remote_groupwareuid_for_message(Node, SharedPrefix, PathDelim, FolderUri, MessageUid) ->
+    spawn_link(Node, ?MODULE, async_groupwareuid_for_message, [self(), SharedPrefix, PathDelim, FolderUri, MessageUid]),
+    %%TODO: error on timeout
+    receive
+        { payload_for_message, Response } -> Response
+    end.
+
+-spec async_groupwareuid_for_message(Pid :: pid(), SharedPrefix :: string(), PathDelim :: string(), FolderUri :: binary(), MessageUid :: integer()) -> binary().
+async_groupwareuid_for_message(Pid, SharedPrefix, PathDelim, FolderUri, MessageUid) ->
+    Pid ! groupwareuid_for_message(SharedPrefix, PathDelim, FolderUri, MessageUid).
+
+-spec groupwareuid_for_message(SharedPrefix :: string(), PathDelim :: string(), FolderUri :: binary(), MessageUid :: integer()) -> binary().
+groupwareuid_for_message(SharedPrefix, PathDelim, FolderUri, MessageUid) ->
+    %%TODO determine if this can / should be cached to avoid repetition
+    FsPath = filesystem_folder_path(SharedPrefix, PathDelim, FolderUri),
+    FsFilePath = FsPath ++ "/" ++ integer_to_list(MessageUid) ++ ".",
+    find_groupwareuid_in_file(file:open(FsFilePath, [read, raw, binary])).
+
+-spec filesystem_folder_path(SharedPrefix :: string(), PathDelim :: string(), FolderUri :: binary()) -> string().
+filesystem_folder_path(SharedPrefix, PathDelim, FolderUri) ->
+    FolderPath = http_uri:decode(binary_to_list(eimap_utils:extract_path_from_uri(SharedPrefix, PathDelim, FolderUri))),
+    FsPathRaw = os:cmd(["/usr/lib/cyrus-imapd/mbpath", " ", FolderPath]),
+    string:strip(FsPathRaw, right, $\n).
+
+-spec find_groupwareuid_in_file({ ok, IoDevice :: any() } | any()) -> binary().
+find_groupwareuid_in_file({ ok, IoDevice }) ->
+    GroupwareUid = find_groupwareuid_in_file(IoDevice, unknown, unknown, file:read_line(IoDevice)),
+    file:close(IoDevice),
+    GroupwareUid;
+find_groupwareuid_in_file(_) ->
+    %% could not open file, just return nothing
+    <<"">>.
+
+-spec find_groupwareuid_in_file(IoDevice :: any(), Type :: unknown | binary(), Subject :: unknown | binary(), Line :: { ok, binary()} | { error, binary()} | eof) -> binary().
+find_groupwareuid_in_file(IoDevice, _Type, Subject, { ok, <<"X-Kolab-Type: ", Value/binary>>}) ->
+    case Subject of
+        unknown -> find_groupwareuid_in_file(IoDevice, Value, Subject, file:read_line(IoDevice));
+        _ -> Subject
+    end;
+find_groupwareuid_in_file(IoDevice, Type, _Subject, { ok, <<"Subject: ", Value/binary>> }) ->
+    case Type of
+        unknown -> find_groupwareuid_in_file(IoDevice, Type, Value, file:read_line(IoDevice));
+        _ -> binary:part(Value, 0, size(Value) - 1)
+    end;
+find_groupwareuid_in_file(_IoDevice, _Type, _Subject, { ok, <<"--=", _Value/binary>>}) ->
+    <<"">>;
+find_groupwareuid_in_file(IoDevice, Type, Subject, { ok, _Line }) ->
+    find_groupwareuid_in_file(IoDevice, Type, Subject, file:read_line(IoDevice));
+find_groupwareuid_in_file(_IoDevice, _Type, _Subject, _) ->
+    <<"">>.
+

egara-0.2.tar.gz/apps/egara/src/lib Deleted

egara-0.2.tar.gz/apps/egara/src/lib/egara_history_entry.erl Deleted

@@ -1,26 +0,0 @@
--module(egara_history_entry).
--export([store/7]).
-
-store(_Storage, _Timestamp, _GroupwareUid, _NewFolderUid, none, _OldFolderUid, _OldMessageUid) ->
-    ok;
-store(Storage, Timestamp, GroupwareUid, NewFolderUid, NewMessageUid, OldFolderUid, none) ->
-    NewMessageUidBin = integer_to_binary(NewMessageUid),
-    Key = generate_history_key(Storage, Timestamp, GroupwareUid, NewFolderUid, NewMessageUidBin),
-    ValueTerm = [{ <<"groupware_uid">>, GroupwareUid }, { <<"history">>, [] }],
-    ValueJson = jsx:encode(ValueTerm),
-    egara_storage:store_message_history_entry(Storage, Key, ValueJson);
-store(Storage, Timestamp, GroupwareUid, NewFolderUid, NewMessageUid, OldFolderUid, OldMessageUid) ->
-    NewMessageUidBin = integer_to_binary(NewMessageUid),
-    OldMessageUidBin = integer_to_binary(OldMessageUid),
-    Key = generate_history_key(Storage, Timestamp, GroupwareUid, NewFolderUid, NewMessageUidBin),
-    ValueTerm = [{<<"groupware_uid">>, GroupwareUid }, { <<"history">>, [{ <<"imap">>, [{ <<"previous_id">>, OldMessageUidBin }, { <<"previous_folder">>, OldFolderUid }] }] }],
-    ValueJson = jsx:encode(ValueTerm),
-    egara_storage:store_message_history_entry(Storage, Key, ValueJson).
-
-%% PRIVATE
-generate_history_key(_, Timestamp, undefined, FolderUid, MessageUid) ->
-    <<"message::", FolderUid/binary, "::", MessageUid/binary, "::", Timestamp/binary, "::">>;
-generate_history_key(_, Timestamp, GroupwareUid, FolderUid, MessageUid) ->
-    <<"message::", FolderUid/binary, "::", MessageUid/binary, "::", Timestamp/binary, "::", GroupwareUid/binary>>.
-
-

egara-0.2.tar.gz/apps/egara/src/lib/egara_notification_queue.erl Deleted

@@ -1,258 +0,0 @@
-%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
-%%
-%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
-%%
-%% This program is free software: you can redistribute it and/or modify
-%% it under the terms of the GNU General Public License as published by
-%% the Free Software Foundation, either version 3 of the License, or
-%% (at your option) any later version.
-%%
-%% This program is distributed in the hope that it will be useful,
-%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-%% GNU General Public License for more details.
-%%
-%% You should have received a copy of the GNU General Public License
-%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
--module(egara_notification_queue).
--export([ install/1, start/0,
-          notification/1, next_unassigned/0, process_next_unassigned/2,
-          add/2, add/3, remove/1, remove_all/0,
-          assign/2, assign_next/1, assigned_to/1,
-          release/1, release/2, release_orphaned/0, release_all/0,
-          migrate_failures/0,
-          max_key/0]).
--include_lib("stdlib/include/qlc.hrl").
--record(egara_incoming_notification, { id, claimed = 0, fails = 0, term }).
--record(egara_incoming_notification_fails, { id, term }).
--define(MAX_FAILURES, 5).
-
-install(Nodes) ->
-    rpc:multicall(Nodes, application, stop, [mnesia]),
-    %% TODO: tune mnesia for large batches of writes; see -> http://streamhacker.com/2008/12/10/how-to-eliminate-mnesia-overload-events/
-    mnesia:create_schema(Nodes),
-    rpc:multicall(Nodes, application, start, [mnesia]),
-    %% create index for claimed
-    try
-        mnesia:create_table(egara_incoming_notification,
-                            [{ attributes, record_info(fields, egara_incoming_notification) },
-                             { type, ordered_set },
-                             { disc_copies, Nodes }]),
-        mnesia:create_table(egara_incoming_notification_fails,
-                            [{ attributes, record_info(fields, egara_incoming_notification_fails) },
-                             { disc_copies, Nodes }])
-    of _ -> ok
-    catch
-        error:_ -> ok
-    end.
-
-start() ->
-    mnesia:wait_for_tables([egara_incoming_notification], 5000).
-
-add(Key, Term) ->
-    F = fun() ->
-                Rec = #egara_incoming_notification{ id = Key, term = Term },
-                mnesia:write(Rec)
-        end,
-    mnesia:activity(transaction, F).
-
-add(Key, Term, PID) when is_pid(PID) ->
-    F = fun() ->
-                Rec = #egara_incoming_notification{ id = Key, term = Term, claimed = PID },
-                mnesia:write(Rec)
-        end,
-    mnesia:activity(transaction, F).
-
-remove(Key) ->
-    F = fun() -> try mnesia:delete({ egara_incoming_notification, Key }) of
-                     _ -> ok
-                 catch
-                     error:Error -> {error, caught, Error}
-                 end
-                 end,
-    mnesia:activity(transaction, F).
-
-remove_all() ->
-    mnesia:clear_table(egara_incoming_notification).
-
-notification(Key) ->
-    F = fun() ->
-                case mnesia:read(egara_incoming_notification, Key) of
-                    [Record] -> Record;
-                    [] -> #egara_incoming_notification{ id = 0 }
-                end
-        end,
-    mnesia:activity(transaction, F).
-
-assign(#egara_incoming_notification{ id = Key }, PID) when is_pid(PID) ->
-    assign(Key, PID);
-
-assign(Key, Fun) when is_function(Fun) ->
-    F = fun() ->
-                QH = qlc:q([ Rec || Rec <- mnesia:table(egara_incoming_notification),
-                             Rec#egara_incoming_notification.fails < ?MAX_FAILURES, Rec#egara_incoming_notification.claimed =:= 0, Rec#egara_incoming_notification.id =:= Key ]),
-                QC = qlc:cursor(QH),
-                Answers = qlc:next_answers(QC, 1),
-                case Answers of
-                    [Record|_] -> mnesia:write(Record#egara_incoming_notification{ claimed = spawn(Fun) });
-                    _ -> notfound
-                end
-        end,
-    mnesia:activity(transaction, F);
-assign(Key, PID) when is_pid(PID) ->
-    F = fun() ->
-                QH = qlc:q([ Rec || Rec <- mnesia:table(egara_incoming_notification),
-                             Rec#egara_incoming_notification.fails < ?MAX_FAILURES, Rec#egara_incoming_notification.claimed =:= 0, Rec#egara_incoming_notification.id =:= Key ]),
-                QC = qlc:cursor(QH),
-                Answers = qlc:next_answers(QC, 1),
-                case Answers of
-                    [Record|_] -> mnesia:write(Record#egara_incoming_notification{ claimed = PID });
-                    _ -> notfound
-                end
-        end,
-    mnesia:activity(transaction, F).
-
-release(Key, PID) ->
-    %%TODO: make this more efficient by using qlc; currently scans whole table!
-    Pattern = #egara_incoming_notification{ _ = '_', claimed = PID, id = Key },
-    F = fun() ->
-                %% check if claimed is set and if so if the process is still running
-                case mnesia:match_object(Pattern) of
-                    [#egara_incoming_notification{ term = Term }] -> mnesia:write(#egara_incoming_notification{ id = Key, term = Term, claimed = 0 });
-                    [] -> { error, notfound}
-                end
-        end,
-    mnesia:activity(transaction, F).
-
-release(Key) ->
-    F = fun() ->
-                case mnesia:read(egara_incoming_notification, Key) of
-                    [#egara_incoming_notification{ term = Term }] -> mnesia:write(#egara_incoming_notification{ id = Key, term = Term, claimed = 0 }), ok;
-                    [] -> false
-                end
-        end,
-    mnesia:activity(transaction, F).
-
-release_orphaned() ->
-    F = fun() ->
-                QH = qlc:q([ Record || #egara_incoming_notification{ claimed = PID } = Record <- mnesia:table(egara_incoming_notification),
-                             is_pid(PID), process_info(PID) =:= undefined]),
-                qlc:fold(fun(Record, N) ->
-                                 Failed = Record#egara_incoming_notification.fails + 1,
-                                 mnesia:write(Record#egara_incoming_notification{ claimed = 0, fails = Failed }), N + 1 end,
-                         0, QH)
-        end,
-    mnesia:activity(transaction, F).
-
-migrate_failures() ->
-    F = fun() ->
-                Timestamp = egara_utils:current_timestamp(),
-                QH = qlc:q([ Record || #egara_incoming_notification{ fails = Fails } = Record <- mnesia:table(egara_incoming_notification), Fails >= ?MAX_FAILURES ]),
-                qlc:fold(fun(#egara_incoming_notification{ id = Key, term = Term } = Record, N) ->
-                                 IntBin = integer_to_binary(N),
-                                 mnesia:write(#egara_incoming_notification_fails{ id = <<Timestamp/binary, "_", IntBin/binary>>,
-                                                                                  term = Term }),
-                                 remove(Key),
-                                 N + 1 end,
-                         0, QH)
-        end,
-    mnesia:activity(transaction, F).
-
-release_all() ->
-    F = fun() ->
-                QH = qlc:q([ Record || #egara_incoming_notification{ claimed = PID } = Record <- mnesia:table(egara_incoming_notification),
-                             is_pid(PID) ]),
-                qlc:fold(fun(Record, N) -> mnesia:write(Record#egara_incoming_notification{ claimed = 0 }), N + 1 end, 0, QH)
-        end,
-    mnesia:activity(transaction, F).
-
-assigned_to(Key) ->
-    F = fun() ->
-                QH = qlc:q([ Rec || Rec <- mnesia:table(egara_incoming_notification),
-                             Rec#egara_incoming_notification.id =:= Key]),
-                QC = qlc:cursor(QH),
-                Answers = qlc:next_answers(QC, 1),
-                case Answers of
-                    [Record|_] -> Record#egara_incoming_notification.claimed;
-                    _ -> notfound
-                end
-        end,
-    mnesia:activity(transaction, F).
-
-assign_next(PID) when is_pid(PID) ->
-    F = fun() ->
-                QH = qlc:q([ Rec || Rec <- mnesia:table(egara_incoming_notification),
-                             Rec#egara_incoming_notification.claimed =:= 0, Rec#egara_incoming_notification.fails < ?MAX_FAILURES]),
-                QC = qlc:cursor(QH),
-                Answers = qlc:next_answers(QC, 1),
-                case Answers of
-                    [Record|_] -> mnesia:write(Record#egara_incoming_notification{ claimed = PID }), { Record#egara_incoming_notification.id, Record#egara_incoming_notification.term };
-                    _ -> notfound
-                end
-        end,
-    mnesia:activity(transaction, F).
-
-%% RequestedN -> the max number of notifications to process
-%% C -> continuation to call with each notification
-%% N -> countdown to 0
-%% [] -> List of matching objects

egara-0.2.tar.gz/apps/egara/src/lib/egara_utils.erl Deleted

@@ -1,22 +0,0 @@
-%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
-%%
-%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
-%%
-%% This program is free software: you can redistribute it and/or modify
-%% it under the terms of the GNU General Public License as published by
-%% the Free Software Foundation, either version 3 of the License, or
-%% (at your option) any later version.
-%%
-%% This program is distributed in the hope that it will be useful,
-%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-%% GNU General Public License for more details.
-%%
-%% You should have received a copy of the GNU General Public License
-%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
--module(egara_utils).
--export([ current_timestamp/0, normalize_timestamp/1 ]).
-
-current_timestamp() -> iso8601:format(os:timestamp()).
-normalize_timestamp(Timestamp) -> iso8601:format(iso8601:parse_exact(Timestamp)).

egara-0.2.tar.gz/apps/egara/src/lib/imap Deleted

egara-0.2.tar.gz/apps/egara/src/lib/imap/egara_imap_command_annotation.erl Deleted

@@ -1,75 +0,0 @@
-%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
-%%
-%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
-%%
-%% This program is free software: you can redistribute it and/or modify
-%% it under the terms of the GNU General Public License as published by
-%% the Free Software Foundation, either version 3 of the License, or
-%% (at your option) any later version.
-%%
-%% This program is distributed in the hope that it will be useful,
-%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-%% GNU General Public License for more details.
-%%
-%% You should have received a copy of the GNU General Public License
-%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
--module(egara_imap_command_annotation).
--export([new/1, parse/2]).
-
-%% Public API
-new(Folder) when is_binary(Folder) ->
-    <<"GETANNOTATION \"", Folder/binary, "\" \"*\" \"value.shared\"">>.
-
-parse(Data, Tag) when is_binary(Data) ->
-    case egara_imap_utils:check_response_for_failure(Data, Tag) of
-        ok ->
-            Lines = binary:split(Data, <<"\r\n">>, [global]),
-            Rv = lists:foldl(fun(Line, Acc) -> parseLine(Line, Acc, Tag) end, [], Lines),
-            { fini, Rv };
-        { _, Reason } ->
-            { error, Reason }
-    end.
-
-
-%% Private API
-parseLine(<<" ", Data/binary>>, Acc, Tag) ->
-    parseLine(Data, Acc, Tag);
-parseLine(<<"* ANNOTATION ", Data/binary>>, Acc, _Tag) ->
-    Pieces = binary:split(Data, <<"\"">>, [global]),
-    process_pieces(Pieces, Acc);
-parseLine(<<>>, Acc, _Tag) ->
-    Acc;
-parseLine(Data, Acc, Tag) ->
-    case binary:match(Data, Tag) of
-        { 0, End } -> handle_possible_end(binary:part(Data, End + 1, byte_size(Data) - End - 1), Acc);
-        _ -> lager:warning("Unexpected response from imap server: ~p", [Data]), Acc
-    end.
-
-process_pieces([MBox, Key, _, _, _, Value, _], Acc) when MBox =/= <<>> -> [ { Key, translate(Value) } | Acc ];
-process_pieces([_, _MBox, _, Key, _, _, _, Value, _], Acc) -> [ { Key, translate(Value) } | Acc ];
-process_pieces(_, Acc) -> Acc.
-
-translate(<<"false">>) -> false;
-translate(<<"true">>) -> true;
-translate(Value) ->
-    try list_to_integer(binary_to_list(Value)) of
-        Number -> Number
-    catch
-        _:_ -> Value
-    end.
-
-handle_possible_end(<<"OK", _/binary>>, Acc) ->
-    Acc;
-handle_possible_end(<<"NO ", Reason/binary>>, Acc) ->
-    lager:warning("Annotation error from imap server: ~p", [Reason]),
-    Acc;
-handle_possible_end(<<"BAD ", Reason/binary>>, Acc) ->
-    lager:warning("Annotation error from imap server: ~p", [Reason]),
-    Acc;
-handle_possible_end([_Tag, Message], Acc) ->
-    handle_possible_end(Message, Acc);
-handle_possible_end(Message, Acc) ->
-    lager:warning("Unexpected response from imap server: ~p", [Message]),
-    Acc.

egara-0.2.tar.gz/apps/egara/src/lib/imap/egara_imap_command_examine.erl Deleted

@@ -1,32 +0,0 @@
-%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
-%%
-%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
-%%
-%% This program is free software: you can redistribute it and/or modify
-%% it under the terms of the GNU General Public License as published by
-%% the Free Software Foundation, either version 3 of the License, or
-%% (at your option) any later version.
-%%
-%% This program is distributed in the hope that it will be useful,
-%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-%% GNU General Public License for more details.
-%%
-%% You should have received a copy of the GNU General Public License
-%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
--module(egara_imap_command_examine).
--export([new/1, parse/2]).
-
-%% https://tools.ietf.org/html/rfc3501#section-6.3.2
-
-%% Public API
-new(MBox) when is_binary(MBox) -> <<"EXAMINE \"", MBox/binary, "\"">>.
-
-parse(Data, Tag) ->
-    case egara_imap_utils:check_response_for_failure(Data, Tag) of
-        ok -> { fini, ok };
-        { _, Reason } -> { error, Reason }
-    end.
-
-%% Private API

egara-0.2.tar.gz/apps/egara/src/lib/imap/egara_imap_command_namespace.erl Deleted

@@ -1,68 +0,0 @@
-%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
-%%
-%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
-%%
-%% This program is free software: you can redistribute it and/or modify
-%% it under the terms of the GNU General Public License as published by
-%% the Free Software Foundation, either version 3 of the License, or
-%% (at your option) any later version.
-%%
-%% This program is distributed in the hope that it will be useful,
-%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-%% GNU General Public License for more details.
-%%
-%% You should have received a copy of the GNU General Public License
-%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
--module(egara_imap_command_namespace).
--export([new/0, parse/2]).
-
-%% http://tools.ietf.org/html/rfc2342
-
-%% Public API
-new() -> <<"NAMESPACE">>.
-
-parse(<<"* NAMESPACE ", Data/binary>>, _Tag) ->
-    NamespaceParts = process_shared_prefix_parts(Data, 1),
-    { fini, relevant_shared_prefix_parts(NamespaceParts) }.
-
-%% Private API
-relevant_shared_prefix_parts([]) -> { none, none};
-relevant_shared_prefix_parts([[], [], _]) -> { none, none };
-relevant_shared_prefix_parts([[], Delim]) -> { none, Delim };
-relevant_shared_prefix_parts([SharedPrefix, [], _]) -> { SharedPrefix, "/" };
-relevant_shared_prefix_parts([SharedPrefix, Delim]) -> { SharedPrefix, Delim }.
-
-process_shared_prefix_parts(<<"NIL ", Data/binary>>, PartNumber) ->
-    process_shared_prefix_parts(Data, PartNumber + 1);
-process_shared_prefix_parts(<<"((", Data/binary>>, PartNumber) ->
-    process_shared_prefix_parts_inner(Data, [], PartNumber);
-process_shared_prefix_parts(<<>>, _PartNumber) -> [].
-
-process_shared_prefix_parts_inner(<<"))", _/binary>>, Acc, 3) ->
-    parse_shared_prefix_fields(lists:reverse(Acc), []);
-process_shared_prefix_parts_inner(<<")) ", Data/binary>>, _Acc, N) ->
-    process_shared_prefix_parts(Data, N + 1);
-process_shared_prefix_parts_inner(<<Char, Data/binary>>, Acc, 3) ->
-    process_shared_prefix_parts_inner(Data, [Char|Acc], 3);
-process_shared_prefix_parts_inner(<<Char, Data/binary>>, Acc, N) ->
-    process_shared_prefix_parts_inner(Data, Acc, N);
-process_shared_prefix_parts_inner(<<>>, _Acc, _N) ->
-    [].
-
-%%FIXME: naively assumes no "s in the values are allowed. is this correct?
-parse_shared_prefix_fields([$"|Tail], Acc) ->
-    parse_shared_prefix_field_inner(Tail, [], Acc);
-parse_shared_prefix_fields([_|Tail], Acc) ->
-    parse_shared_prefix_fields(Tail, Acc);
-parse_shared_prefix_fields([], Acc) ->
-    lists:reverse(Acc).
-
-parse_shared_prefix_field_inner([$"|Tail], PartAcc, Acc) ->
-    parse_shared_prefix_fields(Tail, [lists:reverse(PartAcc)|Acc]);
-parse_shared_prefix_field_inner([Char|Tail], PartAcc, Acc) ->
-    parse_shared_prefix_field_inner(Tail, [Char|PartAcc], Acc);
-parse_shared_prefix_field_inner([], _PartAcc, Acc) ->
-    Acc.
-

egara-0.2.tar.gz/apps/egara/src/lib/imap/egara_imap_command_peek_message.erl Deleted

@@ -1,136 +0,0 @@
-%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
-%%
-%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
-%%
-%% This program is free software: you can redistribute it and/or modify
-%% it under the terms of the GNU General Public License as published by
-%% the Free Software Foundation, either version 3 of the License, or
-%% (at your option) any later version.
-%%
-%% This program is distributed in the hope that it will be useful,
-%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-%% GNU General Public License for more details.
-%%
-%% You should have received a copy of the GNU General Public License
-%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
--module(egara_imap_command_peek_message).
--export([new/1, parse/2, continue_parse/3]).
--record(parse_state, { body_size, parts, data }).
--record(parts, { headers = <<"">>, body = <<"">>, flags = <<"">> }).
-
-%% https://tools.ietf.org/html/rfc3501#section-6.4.5
-
-%% Public API
-new(MessageID) when is_integer(MessageID) -> new(integer_to_binary(MessageID));
-new(MessageID) when is_binary(MessageID) -> <<"UID FETCH ",  MessageID/binary, " (FLAGS BODY.PEEK[HEADER] BODY.PEEK[TEXT])">>.
-
-continue_parse(Data, _Tag, #parse_state{ body_size = Size, parts = Parts, data = PrevData }) ->
-    try_body_parse(<<PrevData/binary, Data/binary>>, Size, Parts).
-
-parse(Data, Tag) when is_binary(Data) ->
-    case egara_imap_utils:check_response_for_failure(Data, Tag) of
-        ok -> process_parts(get_past_headers(Data));
-        { _, Reason } -> { error, Reason }
-    end.
-
-%% Private API
-process_parts(#parts{ headers = Headers, flags = Flags, body = Body }) ->
-    { fini, [ { flags, binary:split(Flags, <<" ">>, [global]) },
-              { headers, filter_headers(binary:split(Headers, <<"\r\n">>, [global])) },
-              { message, <<Headers/binary, Body/binary>> } ] };
-process_parts(Result) ->
-    Result.
-
-get_past_headers(<<" OK ", _Data/binary>>) -> #parts{};
-get_past_headers(<<" FETCH ", Data/binary>>) -> find_open_parens(Data);
-get_past_headers(<<_, Data/binary>>) -> get_past_headers(Data);
-get_past_headers(<<>>) -> { error, <<"Unparsable">> }.
-
-find_open_parens(<<$(, Data/binary>>) -> parse_next_component(Data, #parts{});
-find_open_parens(<<_, Data/binary>>) -> find_open_parens(Data).
-
-parse_next_component(<<"FLAGS (", Data/binary>>, Parts) ->
-    parse_flags(Data, Parts);
-parse_next_component(<<"BODY[HEADER] {", Data/binary>>, Parts) ->
-    parse_header(Data, Parts);
-parse_next_component(<<"BODY[TEXT] {", Data/binary>>, Parts) ->
-    parse_body(Data, Parts);
-parse_next_component(<<_, Data/binary>>, Parts) -> parse_next_component(Data, Parts);
-parse_next_component(<<"OK Completed", _/binary>>, Parts) -> Parts;
-parse_next_component(<<>>, Parts) -> Parts.
-
-parse_flags(Data, Parts) -> parse_flags(Data, Parts, Data, 0).
-parse_flags(OrigData, Parts, <<$), Rest/binary>>, Length) ->
-    FlagsString = binary:part(OrigData, 0, Length),
-    parse_next_component(Rest, Parts#parts{ flags = FlagsString });
-parse_flags(OrigData, Parts, <<_, Data/binary>>, Length) ->
-    parse_flags(OrigData, Parts, Data, Length + 1);
-parse_flags(_OrigData, Parts, <<>>, _Length) ->
-    Parts.
-
-parse_header(Data, Parts) -> parse_header(Data, Parts, Data, 0).
-parse_header(_OrigData, Parts, <<$}, Rest/binary>>, 0) ->
-    parse_next_component(Rest, Parts);
-parse_header(OrigData, Parts, <<$}, Rest/binary>>, Length) ->
-    ByteSizeString = binary:part(OrigData, 0, Length),
-    Size = binary_to_integer(ByteSizeString),
-    HeaderString = binary:part(Rest, 2, Size), %% the 2 is for \r\n
-    %%FIXME: make sure we have enough data loaded, otherwise continue
-    Remainder = binary:part(Rest, Size, byte_size(Rest) - Size),
-    parse_next_component(Remainder, Parts#parts{ headers = HeaderString });
-parse_header(OrigData, Parts, <<_, Rest/binary>>, Length) ->
-    parse_header(OrigData, Parts, Rest, Length + 1);
-parse_header(_OrigData, Parts, <<>>, _Length) ->
-    Parts.
-
-parse_body(Data, Parts) -> parse_body(Data, Parts, Data, 0).
-parse_body(_OrigData, Parts, <<$}, Rest/binary>>, 0) ->
-    parse_next_component(Rest, Parts);
-parse_body(OrigData, Parts, <<$}, Rest/binary>>, Length) ->
-    ByteSizeString = binary:part(OrigData, 0, Length),
-    Size = binary_to_integer(ByteSizeString),
-    %%lager:info("We have ... ~p ~p ~p", [ByteSizeString, Size, byte_size(Rest)]),
-    try_body_parse(Rest, Size, Parts);
-parse_body(OrigData, Parts, <<_, Rest/binary>>, Length) ->
-    parse_body(OrigData, Parts, Rest, Length + 1);
-parse_body(_OrigData, Parts, <<>>, _Length) ->
-    Parts.
-
-try_body_parse(Data, Size, Parts) ->
-    case Size > byte_size(Data) of
-        true ->
-            { more, fun ?MODULE:continue_parse/3, #parse_state{ body_size = Size, parts = Parts, data = Data } };
-        false ->
-            Body = binary:part(Data, 2, Size), %% the 2 is for \r\n
-            Remainder = binary:part(Data, Size, byte_size(Data) - Size),
-            process_parts(parse_next_component(Remainder, Parts#parts{ body = Body }))
-    end.
-
-filter_headers(RawHeaders) ->
-    filter_headers(RawHeaders, none, none, []).
-
-filter_headers([], CurrentKey, CurrentValue, Acc) ->
-    filter_header_add(CurrentKey, CurrentValue, Acc);
-filter_headers([<<>>|Headers], CurrentKey, CurrentValue, Acc) ->
-    filter_headers(Headers, CurrentKey, CurrentValue, Acc);
-filter_headers([Header|Headers], CurrentKey, CurrentValue, Acc) ->
-    filter_header(Headers, CurrentKey, CurrentValue, Acc, binary:split(Header, <<": ">>)).
-
-filter_header(Headers, CurrentKey, CurrentValue, Acc, [Key, Value]) ->
-    Added = filter_header_add(CurrentKey, CurrentValue, Acc),
-    filter_headers(Headers, Key, Value, Added);
-filter_header(Headers, none, _CurrentValue, Acc, _Value) ->
-    filter_headers(Headers, none, none, Acc);
-filter_header(Headers, CurrentKey, CurrentValue, Acc, [Value]) ->
-    %%TODO: get rid of tab, ensure "proper" whitespace
-    NewValue = <<CurrentValue/binary, Value/binary>>,
-    filter_headers(Headers, CurrentKey, NewValue, Acc).
-
-filter_header_add(CurrentKey, CurrentValue, Acc) when CurrentKey =/= none, CurrentValue =/= none ->
-    %%TODO: split CurrentValue on ','
-    [ { CurrentKey, CurrentValue } | Acc ];
-filter_header_add(_, _, Acc) ->
-    Acc.
-

egara-0.2.tar.gz/apps/egara/src/lib/imap/egara_imap_uidset.erl Deleted

@@ -1,53 +0,0 @@
-%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
-%%
-%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
-%%
-%% This program is free software: you can redistribute it and/or modify
-%% it under the terms of the GNU General Public License as published by
-%% the Free Software Foundation, either version 3 of the License, or
-%% (at your option) any later version.
-%%
-%% This program is distributed in the hope that it will be useful,
-%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-%% GNU General Public License for more details.
-%%
-%% You should have received a copy of the GNU General Public License
-%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
--module(egara_imap_uidset).
--export([parse/1, next_uid/1]).
-
-%% tokens have the form: integer | { integer, integer }
--record(uidset, { tokens = [], current = none }).
-
-%% currently do not support parsing regular (list) strings, but could be added
-parse(UidSet) when is_binary(UidSet) ->
-    Components = binary:split(UidSet, <<",">>, [global]),
-    try lists:foldl(fun(Component, Acc) -> add_component(binary:split(Component, <<":">>), Acc) end, [], Components) of
-        Parsed -> #uidset{ tokens = lists:reverse(Parsed) }
-    catch
-        _:_ -> badarg
-    end.
-
-add_component([First, Second], Acc) -> add_component(binary_to_integer(First), binary_to_integer(Second), Acc);
-add_component([<<"">>], Acc) -> Acc;
-add_component([Single], Acc) when is_binary(Single) -> add_component(binary_to_integer(Single), Acc);
-add_component(Uid, Acc) when is_integer(Uid) -> [Uid|Acc];
-add_component(_, _Acc) -> throw(badarg).
-
-add_component(First, Second, Acc) when is_integer(First), is_integer(Second) -> add_range(First, Second, Acc);
-add_component(_, _, _Acc) -> throw(badarg).
-
-add_range(First, Second, Acc) when First == Second -> [First|Acc];
-add_range(First, Second, Acc) -> [{ First, Second }|Acc].
-
-next_uid(#uidset{ tokens = Tokens, current = Current }) -> next_uid(Tokens, Current).
-next_uid([], _Current) -> { none, #uidset {} };
-next_uid([{ First, Second }|_] = FullList, Current) -> next_in_range(FullList, First, Second, Current);
-next_uid([Uid|UidSet], _Current) -> { Uid, #uidset{ tokens = UidSet } }.
-
-next_in_range(Tokens, First, _Second, none) -> { First, #uidset{ tokens = Tokens, current = First } };
-next_in_range(Tokens, First, Second, Current) when First < Second, Current < Second -> Next = Current + 1, { Next, #uidset{ tokens = Tokens, current = Next } };
-next_in_range(Tokens, First, Second, Current) when First > Second, Current > Second -> Next = Current - 1, { Next, #uidset{ tokens = Tokens, current = Next } };
-next_in_range([_|Tokens], _First, _Second, _Current) -> next_uid(#uidset{ tokens = Tokens }).

egara-0.2.tar.gz/apps/egara/src/lib/imap/egara_imap_utils.erl Deleted

@@ -1,99 +0,0 @@
-%% Copyright 2014 Kolab Systems AG (http://www.kolabsys.com)
-%%
-%% Aaron Seigo (Kolab Systems) <seigo a kolabsys.com>
-%%
-%% This program is free software: you can redistribute it and/or modify
-%% it under the terms of the GNU General Public License as published by
-%% the Free Software Foundation, either version 3 of the License, or
-%% (at your option) any later version.
-%%
-%% This program is distributed in the hope that it will be useful,
-%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-%% GNU General Public License for more details.
-%%
-%% You should have received a copy of the GNU General Public License
-%% along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
--module(egara_imap_utils).
--export([
-         extract_path_from_uri/3, extract_uidset_from_uri/1,
-         header_name/1,
-         check_response_for_failure/2
-        ]).
-
-%% Translate the folder name in to a fully qualified folder path such as it
-%% would be used by a cyrus administrator.
-extract_path_from_uri(SharedPrefix, HierarchyDelim, URI) when is_binary(URI) ->
-    extract_path_from_uri(SharedPrefix, HierarchyDelim, binary_to_list(URI));
-extract_path_from_uri(SharedPrefix, HierarchyDelim, URI) when is_list(URI) ->
-    %%lager:info("Parsing ~p", [URI]),
-    SchemeDefaults = [{ imap, 143 }, { imaps, 993 }],
-    ParseOpts = [ { scheme_defaults, SchemeDefaults } ],
-    Path = imap_folder_path(SharedPrefix, HierarchyDelim, http_uri:parse(URI, ParseOpts)),
-    list_to_binary(Path).
-
-extract_uidset_from_uri(URI) when is_binary(URI) ->
-    { TagStart, TagEnd } = binary:match(URI, <<";UID=">>),
-    UIDStart = TagStart + TagEnd + 1,
-    UriLength = byte_size(URI),
-    case binary:match(URI, <<";">>, [{ scope, { UIDStart, UriLength - UIDStart } }]) of
-        nomatch -> binary:part(URI, UIDStart - 1, UriLength - UIDStart + 1);
-        { Semicolon, _ } -> binary:part(URI, UIDStart - 1, Semicolon - UIDStart + 1)
-    end.
-
-header_name(mailbox_uid) -> <<"/vendor/cmu/cyrus-imapd/uniqueid">>;
-header_name(groupware_type) -> <<"X-Kolab-Type">>;
-header_name(groupware_uid) -> <<"Subject">>;
-header_name(_) -> unknown.
-
-check_response_for_failure(Data, Tag) when is_binary(Data), is_binary(Tag) ->
-    NoToken = <<Tag/binary, " NO">>,
-    NoTokenLength = byte_size(NoToken),
-    is_no_token_found(Data, Tag, binary:match(Data, NoToken, [ { scope, { 0, NoTokenLength } } ])).
-
-%% Private
-is_no_token_found(Data, Tag, nomatch) ->
-    BadToken = <<Tag/binary, " BAD">>,
-    BadTokenLength = byte_size(BadToken),
-    is_bad_token_found(Data, Tag, binary:match(Data, BadToken, [ { scope, { 0, BadTokenLength } } ]));
-is_no_token_found(Data, _Tag, { Start, Length }) ->
-    ReasonStart = Start + Length + 1,
-    %% -2 is due to the traling \r\n
-    Reason = binary:part(Data, ReasonStart, byte_size(Data) - ReasonStart - 2),
-    { no, Reason }.
-
-is_bad_token_found(_Data, _Tag, nomatch) ->
-    ok;
-is_bad_token_found(Data, _Tag, { Start, Length }) ->
-    ReasonStart = Start + Length + 1,
-    %% -2 is due to the traling \r\n
-    Reason = binary:part(Data, ReasonStart, byte_size(Data) - ReasonStart - 2),
-    { bad, Reason }.
-
-imap_folder_path_from_parts(none, _HierarchyDelim, none, _Domain, Path) ->
-    Path;
-imap_folder_path_from_parts(SharedPrefix, _HierarchyDelim, none, _Domain, Path) ->
-    case SharedPrefix == string:chars(length(SharedPrefix), Path) of
-        true -> string:sub_str(Path, length(SharedPrefix));
-        _ -> Path
-    end;
-imap_folder_path_from_parts(_SharedPrefix, HierarchyDelim, User, Domain, "INBOX") ->
-    string:join(["user", string:join([User, Domain], "@")], HierarchyDelim);
-imap_folder_path_from_parts(_SharedPrefix, HierarchyDelim, User , Domain, Path) ->
-    string:join(["user", User, string:join([Path, Domain], "@")], HierarchyDelim).
-
-imap_folder_path(_SharedPrefix, _HierarchyDelim, { error, Reason }) ->
-    lager:info("ERROR! ~p", [Reason]),
-    bad_uri;
-imap_folder_path(SharedPrefix, HierarchyDelim, { ok, {_Scheme, User, Domain, _Port, FullPath, _Query} }) ->
-    { VDomain, _ImapHost } = split_imap_uri_domain(string:tokens(Domain, "@")),
-    [ [_|Path] | _ ] = string:tokens(FullPath, ";"),
-    %%lager:info("PARSED IMAP URI: ~p ~p ~p", [User, VDomain, Path]),
-    CanonicalPath = imap_folder_path_from_parts(SharedPrefix, HierarchyDelim, User, VDomain, http_uri:decode(Path)),
-    %%lager:info("PUT TOGETHER AS: ~p", [CanonicalPath]),
-    CanonicalPath.
-
-split_imap_uri_domain([ ImapHost ]) -> { ImapHost, ImapHost };
-split_imap_uri_domain([ VDomain, ImapHost ]) -> { VDomain, ImapHost }.
-

egara-0.2.tar.gz/contrib Added

egara-0.2.tar.gz/contrib/init_riak.sh Added

egara-0.2.tar.gz/deployment Deleted

egara-0.2.tar.gz/deployment/riak.sh Deleted

egara-0.2.tar.gz/deps_diffs/jobs_0.3.diff Deleted

egara-0.2.tar.gz/deps_diffs/protobuffs_0.8.1p4.diff Deleted

egara-0.2.tar.gz/docs/data_storage.md Changed

egara-0.2.tar.gz/docs/egara_architecture.png Added

egara-0.2.tar.gz/priv Added

egara-0.2.tar.gz/priv/sys.config Added

@@ -0,0 +1,118 @@
+%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+
+[
+{ egara,
+  [
+    %% valid values: mail, groupware_object
+    { archival, [groupware_objects] },
+
+    %% { worker_pool_size, 8 }, %% a sensible default is automatically generated if this is left undefined
+
+    %% the events to track; remove any you are not interested in tracking
+    { events_to_track, [
+                { imap_message_event, [
+                                    "FlagsClear",
+                                    "FlagsSet",
+                                    "MessageAppend",
+                                    "MessageCopy",
+                                    "MessageExpire", %% no specimen
+                                    "MessageExpunge",
+                                    "MessageMove",
+                                    "MessageNew",
+                                    "MessageRead",
+                                    "MessageTrash"
+                                  ] },
+                { imap_mailbox_event, [
+                                    "AclChange",
+                                    "MailboxCreate",
+                                    "MailboxDelete",
+                                    "MailboxRename",
+                                    "MailboxSubscribe",
+                                    "MailboxUnSubscribe"
+                                  ] },
+                { imap_session_event, [
+                                    "Login",
+                                    "Logout"
+                                  ] },
+                { imap_quota_event, [
+                                    "QuotaChange", %% no specimen
+                                    "QuotaExceeded", %% no specimen
+                                    "QuotaWithin" %% no specimen
+                                ] }
+              ]
+    },
+
+    %% the list of receivers on this node
+    { receivers, [ cyrus ] },
+
+    %% only needs to be defined if using a non-standard socket and cyrus is in the receivers list
+    { cyrus, [
+                { notification_socket_path, "/tmp/egara-notify" }
+             ]
+    },
+    { imap, [
+                { admin_connection, [
+                                        { host, "192.168.56.101" },
+                                        { port, 993 },
+                                        { tls, true },
+                                        { user, "cyrus-admin" },
+                                        { pass, "12345" }
+                                    ]
+                },
+                { user_blacklist, [
+                                    "cyrus-admin"
+                                  ]
+                }
+            ]
+    },
+    { riak, [
+                { nodes,
+                    [
+                        { "127.0.0.1" , 10017 },
+                        { "127.0.0.1" , 10027 },
+                        { "127.0.0.1" , 10037 },
+                        { "127.0.0.1" , 10047 },
+                        { "127.0.0.1" , 10057 }
+                    ]
+                },
+                { bucket_types,
+                    [
+                        { "egara-unique", [{ last_write_wins, true }, { allow_mult, false }] },
+                        { "egara-lww",    [{ last_write_wins, true }] }
+                    ]
+                }
+            ]
+    },
+    { ldap, [
+             { hosts, [ "192.168.56.101" ]},
+             { bind_dn, "cn=Directory Manager" },
+             { bind_pw, "12345" },
+             { base_dn, "dc=example,dc=org" }
+            ]
+    }
+
+    %% optional archival setting, if set to true than all messages are logged no matter what
+  ]
+},
+{ lager,
+  [
+    {
+        handlers,
+        [
+            { lager_console_backend, info },
+            { lager_file_backend, [ { file, "log/error.log"}, { level, error } ] },
+            { lager_file_backend, [ { file, "log/console.log"}, { level, info } ] }
+        ] }
+  ] },
+{ mnesia,
+    [
+     %%{dir, "db"}
+    ] },
+ {sasl, [
+         {sasl_error_logger, {file, "log/sasl-error.log"}},
+         {errlog_type, error},
+         {error_logger_mf_dir, "log/sasl"},      % Log directory
+         {error_logger_mf_maxbytes, 10485760},   % 10 MB max file size
+         {error_logger_mf_maxfiles, 5}           % 5 files max
+        ]}
+].

egara-0.2.tar.gz/priv/vm.args Added

egara-0.2.tar.gz/rebar Deleted

egara-0.2.tar.gz/rebar.config Changed

@@ -1,29 +1,42 @@
 %% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
 %% ex: ft=erlang ts=4 sw=4 et
 
-{deps_dir, "deps"}.
-{deps, [
-    { jsx, "2.*", { git, "https://github.com/talentdeficit/jsx.git", { tag, "v2.4.0" } } },
-    { lager, ".*", { git, "https://github.com/basho/lager.git", { tag, "2.1.0" } } },
-    %% pull in the proper version of meck before jobs 0.3 gets around to pulling in the wrong version
-    { meck, "0.8.*", {git, "git://github.com/eproxus/meck.git", { tag, "0.8.2"} } },
-    { inert, ".*", { git, "https://github.com/msantos/inert.git" } },
-    { procket, ".*", { git, "https://github.com/msantos/procket.git" } },
-    { riakc, "2.0.1", { git, "git://github.com/basho/riak-erlang-client", { tag, "2.0.1" } } },
-    { poolboy, "1.4.2", { git, "https://github.com/devinus/poolboy.git", { tag, "1.4.2" } } },
-    {'iso8601', ".*",    {git, "git@github.com:aseigo/erlang_iso8601.git"}}
-   ]}.
+{ erl_opts, [
+            { parse_transform, lager_transform },
+            { platform_define, "(linux|solaris|freebsd|darwin)", 'HAVE_SENDFILE' },
+            debug_info,
+            fail_on_warning
+            ]
+}.
 
-{erl_opts, [ %%no_debug_info,
-            {parse_transform, lager_transform},
-            {platform_define, "(linux|solaris|freebsd|darwin)", 'HAVE_SENDFILE'}
-            ]}.
-{sub_dirs, [
-    "apps/egara",
-    "rel"
- ]}.
-{erl_opts, [debug_info, fail_on_warning]}.
+{ deps, [
+    lager,
+    jsx,
+    { lager_syslog, "3.*", { git, "git://github.com/basho/lager_syslog.git", { tag, "3.0.1" } } },
+    { inert, ".*", { git, "https://github.com/msantos/inert.git", { branch, "master" } } }, %{ tag, "0.6.2" } } },
+    { procket, ".*", { git, "https://github.com/msantos/procket.git", { branch, "master" } } }, %{ tag, "0.7.0" } } },
+    { riakc, ".*", { git, "https://github.com/basho/riak-erlang-client", { tag, "2.5.0" } } },
+    poolboy,
+    {'iso8601', ".*", { git, "https://github.com/erlsci/iso8601.git", { tag, "1.2.1" } } },
+    { syn, "1.5.0" },
+    { eimap, ".*", { git, "https://git.kolab.org/diffusion/EI/eimap.git", { tag, "0.4.0" } } }
+   ]
+}.
+
+{ sub_dirs, [ "apps/egara" ] }.
+{ cover_enabled, true }.
+{ relx, [ { release, { egara, "0.1" }, [egara]},
+          { dev_mode, false },
+          { include_erts, false },
+          { extended_start_script, true },
+          { overlay, [
+                      { mkdir, "log/sasl" },
+                      { template, "priv/sys.config", "etc/sys.config" },
+                      { copy, "priv/vm.args", "vm.args" }
+                     ]
+          }
+                                                                                   ]
+}.
 
-%%{require_otp_vsn, "17"}.
 {pre_hooks, [{clean, "rm -fr ebin priv erl_crash.dump"}]}.

egara-0.2.tar.gz/rebar3 Added

egara-0.2.tar.gz/rel Deleted

egara-0.2.tar.gz/rel/reltool.config Deleted

@@ -1,58 +0,0 @@
-%% -*- mode: erlang -*-
-%% ex: ft=erlang
-{sys, [
-       {lib_dirs, ["../apps/egara", "../deps"]},
-       {erts, [{mod_cond, derived}, {app_file, strip}]},
-       {app_file, strip},
-       {rel, "egara", "0.1",
-        [
-         kernel,
-         stdlib,
-         sasl,
-         ssl,
-         jsx,
-         compiler,
-         syntax_tools,
-         goldrush,
-         lager,
-         crypto,
-         procket,
-         inert,
-         mnesia,
-         protobuffs,
-         riak_pb,
-         riakc,
-         poolboy,
-         egara
-        ]},
-       {rel, "start_clean", "",
-        [
-         kernel,
-         stdlib
-        ]},
-       {boot_rel, "egara"},
-       {profile, embedded},
-       {incl_cond, derived},
-       {excl_archive_filters, [".*"]}, %% Do not archive built libs
-       {excl_sys_filters, ["^bin/(?!start_clean.boot)",
-                           "^erts.*/bin/(dialyzer|typer)",
-                           "^erts.*/(doc|info|include|lib|man|src)"]},
-       {excl_app_filters, ["\.gitignore"]},
-       {app, egara, [{mod_cond, app}, {incl_cond, include}, {lib_dir, "../apps/egara"}]}
-      ]}.
-
-{target_dir, "egara"}.
-
-{overlay, [
-           {mkdir, "log/sasl"},
-           {copy, "files/erl", "\{\{erts_vsn\}\}/bin/erl"},
-           {copy, "files/nodetool", "releases/\{\{rel_vsn\}\}/nodetool"},
-           {copy, "egara/bin/start_clean.boot",
-                  "\{\{erts_vsn\}\}/bin/start_clean.boot"},
-           {copy, "files/egara", "bin/egara"},
-           {copy, "files/egara.cmd", "bin/egara.cmd"},
-           {copy, "files/start_erl.cmd", "bin/start_erl.cmd"},
-           {copy, "files/install_upgrade.escript", "bin/install_upgrade.escript"},
-           {copy, "files/sys.config", "releases/\{\{rel_vsn\}\}/sys.config"},
-           {copy, "files/vm.args", "releases/\{\{rel_vsn\}\}/vm.args"}
-          ]}.