Kolab Groupware OBS

Changes of Revision 124

cyrus-imapd-2.5.tar.gz/lib/cyrusdb_skiplist.c Changed

@@ -399,7 +399,7 @@
     case ADD:
 	level = LEVEL_safe(db, ptr);
 	if (!level) {
-	    syslog(LOG_ERR, "IOERROR: skiplist2 RECSIZE_safe not safe %s, offset %u",
+	    syslog(LOG_ERR, "IOERROR: skiplist RECSIZE not safe %s, offset %u",
 		   db->fname, (unsigned)(ptr - db->map_base));
 	    return 0;
 	}
@@ -413,50 +413,20 @@
 	break;
 
     case DELETE:
+	if (!is_safe(db, ptr+8)) {
+	    syslog(LOG_ERR, "IOERROR: skiplist RECSIZE not safe %s, offset %u",
+		   db->fname, (unsigned)(ptr - db->map_base));
+	    return 0;
+	}
 	ret += 8;
 	break;
 
     case COMMIT:
-	ret += 4;
-	break;
-    }
-
-    return ret;
-}
-
-/* how many levels does this record have? */
-static unsigned LEVEL(const char *ptr)
-{
-    const uint32_t *p, *q;
-
-    assert(TYPE(ptr) == DUMMY || TYPE(ptr) == INORDER || TYPE(ptr) == ADD);
-    p = q = (uint32_t *) FIRSTPTR(ptr);
-    while (*p != (uint32_t)-1) p++;
-    return (p - q);
-}
-
-/* how big is this record? */
-static unsigned RECSIZE(const char *ptr)
-{
-    int ret = 0;
-    switch (TYPE(ptr)) {
-    case DUMMY:
-    case INORDER:
-    case ADD:
-	ret += 4;			/* tag */
-	ret += 4;			/* keylen */
-	ret += ROUNDUP(KEYLEN(ptr));    /* key */
-	ret += 4;			/* datalen */
-	ret += ROUNDUP(DATALEN(ptr));   /* data */
-	ret += 4 * LEVEL(ptr);	        /* pointers */
-	ret += 4;			/* padding */
-	break;
-
-    case DELETE:
-	ret += 8;
-	break;
-
-    case COMMIT:
+	if (!is_safe(db, ptr+4)) {
+	    syslog(LOG_ERR, "IOERROR: skiplist RECSIZE not safe %s, offset %u",
+		   db->fname, (unsigned)(ptr - db->map_base));
+	    return 0;
+	}
 	ret += 4;
 	break;
     }
@@ -524,7 +494,12 @@
 }
 
 
-#define PADDING(ptr) (ntohl(*((uint32_t *)((ptr) + RECSIZE(ptr) - 4))))
+static unsigned PADDING_safe(struct dbengine *db, const char *ptr)
+{
+    unsigned size = RECSIZE_safe(db, ptr);
+    if (!size) return 0;
+    return ntohl(*((uint32_t *)((ptr) + size - 4)));
+}
 
 /* given an open, mapped db, read in the header information */
 static int read_header(struct dbengine *db)
@@ -595,9 +570,9 @@
 	       db->fname);
 	r = CYRUSDB_IOERROR;
     }
-    if (!r && LEVEL(dptr) != db->maxlevel) {
+    if (!r && LEVEL_safe(db, dptr) != db->maxlevel) {
 	syslog(LOG_ERR, "DBERROR: %s: DUMMY level(%d) != db->maxlevel(%d)",
-	       db->fname, LEVEL(dptr), db->maxlevel);
+	       db->fname, LEVEL_safe(db, dptr), db->maxlevel);
 	r = CYRUSDB_IOERROR;
     }
 
@@ -1306,7 +1281,7 @@
 	    return CYRUSDB_EXISTS;
 	} else {
 	    /* replace with an equal height node */
-	    lvl = LEVEL(ptr);
+	    lvl = LEVEL_safe(db, ptr);
 
 	    /* log a removal */
 	    WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &delrectype, 4);
@@ -1600,8 +1575,8 @@
 
 	/* find the last log entry */
 	for (offset = tid->logstart, ptr = db->map_base + offset; 
-	     offset + RECSIZE(ptr) != (uint32_t) tid->logend;
-	     offset += RECSIZE(ptr), ptr = db->map_base + offset) ;
+	     offset + RECSIZE_safe(db, ptr) != (uint32_t) tid->logend;
+	     offset += RECSIZE_safe(db, ptr), ptr = db->map_base + offset) ;
 	
 	offset = ptr - db->map_base;
 
@@ -1638,7 +1613,7 @@
 	    /* re-add this record.  it can't exist right now. */
 	    netnewoffset = *((uint32_t *)(ptr + 4));
 	    q = db->map_base + ntohl(netnewoffset);
-	    lvl = LEVEL(q);
+	    lvl = LEVEL_safe(db, q);
 	    (void) find_node(db, KEY(q), KEYLEN(q), updateoffsets);
 	    for (i = 0; i < lvl; i++) {
 		/* the current pointers FROM this node are correct,
@@ -1653,7 +1628,7 @@
 	}
 
 	/* remove looking at this */
-	tid->logend -= RECSIZE(ptr);
+	tid->logend -= RECSIZE_safe(db, ptr);
     }
 
     /* truncate the file to remove log entries */
@@ -1767,13 +1742,13 @@
 	uint32_t netnewoffset;
 
 	ptr = db->map_base + offset;
-	lvl = LEVEL(ptr);
+	lvl = LEVEL_safe(db, ptr);
 	db->listsize++;
 
 	num_iov = 0;
 	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) &iorectype, 4);
 	/* copy all but the rectype from the record */
-	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) ptr + 4, RECSIZE(ptr) - 4);
+	WRITEV_ADD_TO_IOVEC(iov, num_iov, (char *) ptr + 4, RECSIZE_safe(db, ptr) - 4);
 
 	newoffset = lseek(db->fd, 0, SEEK_END);
 	netnewoffset = htonl(newoffset);
@@ -1940,9 +1915,9 @@
 	case INORDER:
 	case ADD:
 	    printf("kl=%d dl=%d lvl=%d\n",
-		   KEYLEN(ptr), DATALEN(ptr), LEVEL(ptr));
+		   KEYLEN(ptr), DATALEN(ptr), LEVEL_safe(db, ptr));
 	    printf("\t");
-	    for (i = 0; i < LEVEL(ptr); i++) {
+	    for (i = 0; i < LEVEL_safe(db, ptr); i++) {
 		printf("%04X ", FORWARD(ptr, i));
 	    }
 	    printf("\n");
@@ -1957,7 +1932,7 @@
 	    break;
 	}
 
-	ptr += RECSIZE(ptr);
+	ptr += RECSIZE_safe(db, ptr);
     }
 
     unlock(db);
@@ -1986,7 +1961,7 @@
 
 	ptr = db->map_base + offset;
 
-	for (i = 0; i < LEVEL(ptr); i++) {
+	for (i = 0; i < LEVEL_safe(db, ptr); i++) {
 	    offset = FORWARD(ptr, i);
 
 	    if (offset > db->map_size) {
@@ -2030,9 +2005,11 @@
 static int recovery(struct dbengine *db, int flags)
 {
     const char *ptr, *keyptr;
+    unsigned filesize = db->map_size;
     unsigned updateoffsets[SKIPLIST_MAXLEVEL+1];
     uint32_t offset, offsetnet, myoff = 0;
-    int r = 0, need_checkpoint = 0;
+    int r = 0;
+    int need_checkpoint = libcyrus_config_getswitch(CYRUSOPT_SKIPLIST_ALWAYS_CHECKPOINT);
     time_t start = time(NULL);
     unsigned i;
 
@@ -2086,11 +2063,11 @@
     }
 
     /* pointers for db->maxlevel */
-    if (!r && LEVEL(ptr) != db->maxlevel) {
+    if (!r && LEVEL_safe(db, ptr) != db->maxlevel) {
 	r = CYRUSDB_IOERROR;
 	syslog(LOG_ERR, 
 	       "DBERROR: skiplist recovery %s: dummy node level: %d != %d",
-	       db->fname, LEVEL(ptr), db->maxlevel);
+	       db->fname, LEVEL_safe(db, ptr), db->maxlevel);
     }
     
     for (i = 0; i < db->maxlevel; i++) {
@@ -2101,7 +2078,7 @@
     
     /* reset the data that was written INORDER by the last checkpoint */
     offset = DUMMY_OFFSET(db) + DUMMY_SIZE(db);
-    while (!r && (offset < db->map_size)
+    while (!r && (offset < filesize)
 	      && TYPE(db->map_base + offset) == INORDER) {
 	ptr = db->map_base + offset;
 	offsetnet = htonl(offset);
@@ -2111,9 +2088,9 @@
 	/* xxx check \0 fill on key */
 
 	/* xxx check \0 fill on data */
-	    
+
 	/* update previous pointers, record these for updating */
-	for (i = 0; !r && i < LEVEL(ptr); i++) {
+	for (i = 0; !r && i < LEVEL_safe(db, ptr); i++) {
 	    r = lseek(db->fd, updateoffsets[i], SEEK_SET);
 	    if (r < 0) {
 		syslog(LOG_ERR, "DBERROR: lseek %s: %m", db->fname);
@@ -2136,15 +2113,23 @@
 	    updateoffsets[i] = offset + (PTR(ptr, i) - ptr);
 	}
 
-	/* check padding */
-	if (!r && PADDING(ptr) != (uint32_t) -1) {
-	    syslog(LOG_ERR, "DBERROR: %s: offset %04X padding not -1",
-		   db->fname, offset);
-	    r = CYRUSDB_IOERROR;
-	}
-
 	if (!r) {
-	    offset += RECSIZE(ptr);
+	    unsigned size = RECSIZE_safe(db, ptr);
+	    if (!size) {
+		syslog(LOG_ERR, "skiplist recovery %s: damaged record at %u, truncating here",
+		       db->fname, offset);
+		filesize = offset;
+		break;
+	    }
+
+	    if (PADDING_safe(db, ptr) != (uint32_t) -1) {
+		syslog(LOG_ERR, "DBERROR: %s: offset %04X padding not -1",
+		       db->fname, offset);
+		filesize = offset;
+		break;
+	    }
+
+	    offset += size;
 	}
     }
 
@@ -2179,7 +2164,7 @@
     }
 
     /* replay the log */
-    while (!r && offset < db->map_size) {
+    while (!r && offset < filesize) {
 	const char *p, *q;
 
 	/* refresh map, so we see the writes we've just done */
@@ -2191,7 +2176,7 @@
 	/* bugs in recovery truncates could have left some bogus zeros here */
 	if (TYPE(ptr) == 0) {
 	    int orig = offset;
-	    while (TYPE(ptr) == 0 && offset < db->map_size) {
+	    while (TYPE(ptr) == 0 && offset < filesize) {
 		offset += 4;
 		ptr = db->map_base + offset;
 	    }
@@ -2204,7 +2189,7 @@
 
 	/* if this is a commit, we've processed everything in this txn */
 	if (TYPE(ptr) == COMMIT) {
-	    offset += RECSIZE(ptr);
+	    offset += RECSIZE_safe(db, ptr);
 	    continue;
 	}
 
@@ -2218,7 +2203,7 @@
 	}
 
 	/* look ahead for a commit */
-	q = db->map_base + db->map_size;
+	q = db->map_base + filesize;
 	p = ptr;
 	for (;;) {
             if (RECSIZE_safe(db, p) <= 0) {
@@ -2230,7 +2215,7 @@
                 p = q;
                 break;
             }
-	    p += RECSIZE(p);
+	    p += RECSIZE_safe(db, p);
 	    if (p >= q) break;
 	    if (TYPE(p) == COMMIT) break;
 	}
@@ -2239,16 +2224,7 @@
 		   "skiplist recovery %s: found partial txn, not replaying",
 		   db->fname);
 
-	    /* no commit, we should truncate */
-	    if (ftruncate(db->fd, offset) < 0) {
-		syslog(LOG_ERR, 
-		       "DBERROR: skiplist recovery %s: ftruncate: %m",
-		       db->fname);
-		r = CYRUSDB_IOERROR;
-	    }
-
-	    /* set the map size back as well */
-	    db->map_size = offset;
+	    filesize = offset;
 
 	    break;
 	}
@@ -2314,7 +2290,7 @@
 	    }
 	    offsetnet = htonl(offset);
 
-	    lvl = LEVEL(ptr);
+	    lvl = LEVEL_safe(db, ptr);
 	    if (lvl > SKIPLIST_MAXLEVEL) {
 		syslog(LOG_ERR,
 		       "DBERROR: skiplist recovery %s: node claims level %d (greater than max %d)",
@@ -2323,15 +2299,15 @@
 	    } else {
 		/* NOTE - in the bogus case where a record with the same key already
 		 * exists, there are three possible cases:
-		 * lvl == LEVEL(keyptr)
+		 * lvl == LEVEL_safe(db, keyptr)
 		 *    * trivial: all to me, all mine to keyptr's FORWARD
-		 * lvl > LEVEL(keyptr)	 -
+		 * lvl > LEVEL_safe(db, keyptr)	 -
 		 *    * all updateoffsets values should point to me
-		 *    * up until LEVEL(keyptr) set to keyptr's next values
+		 *    * up until LEVEL_safe(db, keyptr) set to keyptr's next values
 		 *      (updateoffsets[i] should be keyptr in these cases)
 		 *      then point all my higher pointers are updateoffsets[i]'s
 		 *      FORWARD instead.
-		 * lvl < LEVEL(keyptr)
+		 * lvl < LEVEL_safe(db, keyptr)
 		 *    * updateoffsets values up to lvl should point to me
 		 *    * all mine should point to keyptr's next values
 		 *    * from lvl up, all updateoffsets[i] should point to
@@ -2343,7 +2319,7 @@
 		 */
 		for (i = 0; i < lvl; i++) {
 		    /* set our next pointers */
-		    if (keyptr && i < LEVEL(keyptr)) {
+		    if (keyptr && i < LEVEL_safe(db, keyptr)) {
                         /* need to replace the matching record key */
 			newoffsets[i] = 
 			    htonl(FORWARD(keyptr, i));
@@ -2362,9 +2338,9 @@
 		lseek(db->fd, FIRSTPTR(ptr) - db->map_base, SEEK_SET);
 		retry_write(db->fd, (char *) newoffsets, 4 * lvl);
                 
-		if (keyptr && lvl < LEVEL(keyptr)) {
+		if (keyptr && lvl < LEVEL_safe(db, keyptr)) {
 		    uint32_t newoffsetnet;
-		    for (i = lvl; i < LEVEL(keyptr); i++) {
+		    for (i = lvl; i < LEVEL_safe(db, keyptr); i++) {
 			newoffsetnet = htonl(FORWARD(keyptr, i));
 			/* replace 'updateoffsets' to point onwards */
 			lseek(db->fd, 
@@ -2380,21 +2356,22 @@
 	}
 
 	/* move to next record */
-	offset += RECSIZE(ptr);
+	unsigned size = RECSIZE_safe(db, ptr);
+	if (!size) break;
+	offset += size;
     }
 
-    if (libcyrus_config_getswitch(CYRUSOPT_SKIPLIST_ALWAYS_CHECKPOINT)) {
-	/* refresh map, so we see the writes we've just done */
-	map_refresh(db->fd, 0, &db->map_base, &db->map_len, db->map_size,
-		    db->fname, 0);
-
-	r = mycheckpoint(db);
-
-	if (r || !(flags & RECOVERY_CALLER_LOCKED)) {
-	    unlock(db);
+    /* didn't read the exact end?  We should truncate */
+    if (offset < db->map_size) {
+	if (ftruncate(db->fd, offset) < 0) {
+	    syslog(LOG_ERR,
+		   "DBERROR: skiplist recovery %s: ftruncate: %m",
+		   db->fname);
+	    r = CYRUSDB_IOERROR;
 	}
-    
-	return r;
+
+	/* set the map size back as well */
+	db->map_size = offset;
     }
 
     /* fsync the recovered database */
@@ -2427,13 +2404,16 @@
     }
 
     if (!r && need_checkpoint) {
+	/* refresh map, so we see the writes we've just done */
+	map_refresh(db->fd, 0, &db->map_base, &db->map_len, db->map_size,
+		    db->fname, 0);
 	r = mycheckpoint(db);
     }
 
-    if(r || !(flags & RECOVERY_CALLER_LOCKED)) {
+    if (r || !(flags & RECOVERY_CALLER_LOCKED)) {
 	unlock(db);
     }
-    
+
     return r;
 }

cyrus-imapd-2.5.tar.gz/lib/imapoptions Changed

@@ -169,7 +169,7 @@
    affect LMTP delivery of messages directly to mailboxes via
    plus-addressing. */
 
-{ "annotation_db", "skiplist", STRINGLIST("berkeley", "berkeley-hash", "skiplist", "twoskip")}
+{ "annotation_db", "twoskip", STRINGLIST("berkeley", "berkeley-hash", "skiplist", "twoskip")}
 /* The cyrusdb backend to use for mailbox annotations. */
 
 { "annotation_db_path", NULL, STRING }
@@ -508,7 +508,7 @@
    specifies the actual key used for iSchedule DKIM signing within the
    domain. */
 
-{ "duplicate_db", "skiplist", STRINGLIST("berkeley", "berkeley-nosync", "berkeley-hash", "berkeley-hash-nosync", "skiplist", "sql", "twoskip")}
+{ "duplicate_db", "twoskip", STRINGLIST("berkeley", "berkeley-nosync", "berkeley-hash", "berkeley-hash-nosync", "skiplist", "sql", "twoskip")}
 /* The cyrusdb backend to use for the duplicate delivery suppression
    and sieve. */
 
@@ -1013,10 +1013,10 @@
 { "maxword", 131072, INT }
 /* Maximum size of a single word for the parser.  Default 128k */
 
-{ "mboxkey_db", "skiplist", STRINGLIST("berkeley", "skiplist", "twoskip") }
+{ "mboxkey_db", "twoskip", STRINGLIST("berkeley", "skiplist", "twoskip") }
 /* The cyrusdb backend to use for mailbox keys. */
 
-{ "mboxlist_db", "skiplist", STRINGLIST("flat", "berkeley", "berkeley-hash", "skiplist", "sql", "twoskip")}
+{ "mboxlist_db", "twoskip", STRINGLIST("flat", "berkeley", "berkeley-hash", "skiplist", "sql", "twoskip")}
 /* The cyrusdb backend to use for the mailbox list. */
 
 { "mboxlist_db_path", NULL, STRING }
@@ -1376,7 +1376,7 @@
 /* Unix domain socket that ptloader listens on.
    (defaults to configdir/ptclient/ptsock) */
 
-{ "ptscache_db", "skiplist", STRINGLIST("berkeley", "berkeley-hash", "skiplist", "twoskip")}
+{ "ptscache_db", "twoskip", STRINGLIST("berkeley", "berkeley-hash", "skiplist", "twoskip")}
 /* The cyrusdb backend to use for the pts cache. */
 
 { "ptscache_db_path", NULL, STRING }
@@ -1401,7 +1401,7 @@
 /* This specifies the Class Selector or Differentiated Services Code Point
    designation on IP headers (in the ToS field). */
 
-{ "quota_db", "skiplist", STRINGLIST("flat", "berkeley", "berkeley-hash", "skiplist", "sql", "quotalegacy", "twoskip")}
+{ "quota_db", "twoskip", STRINGLIST("flat", "berkeley", "berkeley-hash", "skiplist", "sql", "quotalegacy", "twoskip")}
 /* The cyrusdb backend to use for quotas. */
 
 { "quota_db_path", NULL, STRING }
@@ -1521,7 +1521,7 @@
    recommended for most cases - it's a good compromise which
    keeps words separate. */
 
-{ "seenstate_db", "skiplist", STRINGLIST("flat", "berkeley", "berkeley-hash", "skiplist", "twoskip")}
+{ "seenstate_db", "twoskip", STRINGLIST("flat", "berkeley", "berkeley-hash", "skiplist", "twoskip")}
 /* The cyrusdb backend to use for the seen state. */
 
 { "sendmail", "/usr/lib/sendmail", STRING }
@@ -1715,7 +1715,7 @@
 { "statuscache", 0, SWITCH }
 /* Enable/disable the imap status cache. */
 
-{ "statuscache_db", "skiplist", STRINGLIST("berkeley", "berkeley-nosync", "berkeley-hash", "berkeley-hash-nosync", "skiplist", "sql", "twoskip") }
+{ "statuscache_db", "twoskip", STRINGLIST("berkeley", "berkeley-nosync", "berkeley-hash", "berkeley-hash-nosync", "skiplist", "sql", "twoskip") }
 /* The cyrusdb backend to use for the imap status cache. */
 
 { "statuscache_db_path", NULL, STRING }
@@ -1880,7 +1880,7 @@
 /* File containing the private key belonging to the certificate in
    tls_server_cert. */
 
-{ "tls_sessions_db", "skiplist", STRINGLIST("berkeley", "berkeley-nosync", "berkeley-hash", "berkeley-hash-nosync", "skiplist", "sql", "twoskip")}
+{ "tls_sessions_db", "twoskip", STRINGLIST("berkeley", "berkeley-nosync", "berkeley-hash", "berkeley-hash-nosync", "skiplist", "sql", "twoskip")}
 /* The cyrusdb backend to use for the TLS cache. */
 
 { "tls_sessions_db_path", NULL, STRING }
@@ -1944,7 +1944,7 @@
    this user.  NOTE: This must be an existing local user name with an
    INBOX, NOT an email address! */
 
-{ "zoneinfo_db", "skiplist", STRINGLIST("flat", "berkeley", "berkeley-hash", "skiplist", "twoskip")}
+{ "zoneinfo_db", "twoskip", STRINGLIST("flat", "berkeley", "berkeley-hash", "skiplist", "twoskip")}
 /* The cyrusdb backend to use for zoneinfo. */
 
 { "zoneinfo_db_path", NULL, STRING }