sin/samba-mirror
1
0
Fork 0
mirror of https://github.com/samba-team/samba.git synced 2025-03-29 02:50:28 +03:00
Code

Make gencache more stable

Browse Source 
This provides a compromise between stability and performance: gencache is a
persistent database these days that for performance reasons can not use tdb
transactions for all writes. This patch splits up gencache into gencache.tdb
and gencache_notrans.tdb. gencache_notrans is used with CLEAR_IF_FIRST, writes
to it don't use transactions. By default every 5 minutes and when a program
exits, all entries from _notrans.tdb are transferred to gencache.tdb in one
transaction.
This commit is contained in:
Volker Lendecke 2009-07-13 17:04:29 +02:00
parent ed87594e5f
commit 8a17cd810f
7 changed files with 299 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
source3/include/proto.h
View File

@ -519,6 +519,7 @@ bool gencache_set(const char *keystr, const char *value, time_t timeout);
bool gencache_del(const char *keystr);
bool gencache_get(const char *keystr, char **valstr, time_t *timeout);
bool gencache_get_data_blob(const char *keystr, DATA_BLOB *blob, bool *expired);
bool gencache_stabilize(void);
bool gencache_set_data_blob(const char *keystr, const DATA_BLOB *blob, time_t timeout);
void gencache_iterate(void (*fn)(const char* key, const char *value, time_t timeout, void* dptr),
void* data, const char* keystr_pattern);

283
source3/lib/gencache.c
View File

@ -32,6 +32,7 @@
#define BLOB_TYPE_LEN 9
static struct tdb_context *cache;
static struct tdb_context *cache_notrans;
/**
* @file gencache.c
@ -52,6 +53,7 @@ static struct tdb_context *cache;
static bool gencache_init(void)
{
char* cache_fname = NULL;
int open_flags = O_RDWR|O_CREAT;
/* skip file open if it's already opened */
if (cache) return True;
@ -60,11 +62,12 @@ static bool gencache_init(void)
DEBUG(5, ("Opening cache file at %s\n", cache_fname));
cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT,
O_RDWR|O_CREAT, 0644);
cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT, open_flags, 0644);
if (!cache && (errno == EACCES)) {
cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT, O_RDONLY, 0644);
open_flags = O_RDONLY;
cache = tdb_open_log(cache_fname, 0, TDB_DEFAULT, open_flags,
0644);
if (cache) {
DEBUG(5, ("gencache_init: Opening cache file %s read-only.\n", cache_fname));
}
@ -74,9 +77,30 @@ static bool gencache_init(void)
DEBUG(5, ("Attempt to open gencache.tdb has failed.\n"));
return False;
}
cache_fname = lock_path("gencache_notrans.tdb");
DEBUG(5, ("Opening cache file at %s\n", cache_fname));
cache_notrans = tdb_open_log(cache_fname, 0, TDB_CLEAR_IF_FIRST,
open_flags, 0644);
if (cache_notrans == NULL) {
DEBUG(5, ("Opening %s failed: %s\n", cache_fname,
strerror(errno)));
tdb_close(cache);
return false;
}
return True;
}
static TDB_DATA last_stabilize_key(void)
{
TDB_DATA result;
result.dptr = (uint8_t *)"@LAST_STABILIZED";
result.dsize = 17;
return result;
}
/**
* Set an entry in the cache file. If there's no such
@ -95,6 +119,13 @@ bool gencache_set(const char *keystr, const char *value, time_t timeout)
int ret;
TDB_DATA databuf;
char* valstr = NULL;
time_t last_stabilize;
if (tdb_data_cmp(string_term_tdb_data(keystr),
last_stabilize_key()) == 0) {
DEBUG(10, ("Can't store %s as a key\n", keystr));
return false;
}
if ((keystr == NULL) || (value == NULL)) {
return false;
@ -112,9 +143,31 @@ bool gencache_set(const char *keystr, const char *value, time_t timeout)
(int)(timeout - time(NULL)),
timeout > time(NULL) ? "ahead" : "in the past"));
ret = tdb_store_bystring(cache, keystr, databuf, 0);
ret = tdb_store_bystring(cache_notrans, keystr, databuf, 0);
SAFE_FREE(valstr);
if (ret != 0) {
return false;
}
/*
* Every 5 minutes, call gencache_stabilize() to not let grow
* gencache_notrans.tdb too large.
*/
last_stabilize = 0;
databuf = tdb_fetch(cache_notrans, last_stabilize_key());
if ((databuf.dptr != NULL)
&& (databuf.dptr[databuf.dsize-1] == '\0')) {
last_stabilize = atoi((char *)databuf.dptr);
SAFE_FREE(databuf.dptr);
}
if ((last_stabilize
+ lp_parm_int(-1, "gencache", "stabilize_interval", 300))
< time(NULL)) {
gencache_stabilize();
}
return ret == 0;
}
@ -129,7 +182,9 @@ bool gencache_set(const char *keystr, const char *value, time_t timeout)
bool gencache_del(const char *keystr)
{
int ret;
bool exists;
bool ret = false;
char *value;
if (keystr == NULL) {
return false;
@ -138,11 +193,46 @@ bool gencache_del(const char *keystr)
if (!gencache_init()) return False;
DEBUG(10, ("Deleting cache entry (key = %s)\n", keystr));
ret = tdb_delete_bystring(cache, keystr);
return ret == 0;
if (tdb_lock_bystring(cache_notrans, keystr) == -1) {
DEBUG(5, ("Could not lock key for %s\n", keystr));
return false;
}
/*
* We delete an element by setting its timeout to 0. This way we don't
* have to do a transaction on gencache.tdb every time we delete an
* element.
*/
exists = gencache_get(keystr, &value, NULL);
if (exists) {
SAFE_FREE(value);
ret = gencache_set(keystr, "", 0);
}
tdb_unlock_bystring(cache_notrans, keystr);
return ret;
}
static bool gencache_pull_timeout(char *val, time_t *pres, char **pendptr)
{
time_t res;
char *endptr;
res = strtol(val, &endptr, 10);
if ((endptr == NULL) || (*endptr != '/')) {
DEBUG(2, ("Invalid gencache data format: %s\n", val));
return false;
}
if (pres != NULL) {
*pres = res;
}
if (pendptr != NULL) {
*pendptr = endptr;
}
return true;
}
/**
* Get existing entry from the cache file.
@ -167,22 +257,29 @@ bool gencache_get(const char *keystr, char **valstr, time_t *timeout)
return false;
}
if (tdb_data_cmp(string_term_tdb_data(keystr),
last_stabilize_key()) == 0) {
DEBUG(10, ("Can't get %s as a key\n", keystr));
return false;
}
if (!gencache_init()) {
return False;
}
databuf = tdb_fetch_bystring(cache, keystr);
databuf = tdb_fetch_bystring(cache_notrans, keystr);
if (databuf.dptr == NULL) {
DEBUG(10, ("Cache entry with key = %s couldn't be found\n",
databuf = tdb_fetch_bystring(cache, keystr);
}
if (databuf.dptr == NULL) {
DEBUG(10, ("Cache entry with key = %s couldn't be found \n",
keystr));
return False;
}
t = strtol((const char *)databuf.dptr, &endptr, 10);
if ((endptr == NULL) || (*endptr != '/')) {
DEBUG(2, ("Invalid gencache data format: %s\n", databuf.dptr));
if (!gencache_pull_timeout((char *)databuf.dptr, &t, &endptr)) {
SAFE_FREE(databuf.dptr);
return False;
}
@ -191,10 +288,21 @@ bool gencache_get(const char *keystr, char **valstr, time_t *timeout)
"timeout = %s", t > time(NULL) ? "valid" :
"expired", keystr, endptr+1, ctime(&t)));
if (t == 0) {
/* Deleted */
SAFE_FREE(databuf.dptr);
return False;
}
if (t <= time(NULL)) {
/* We're expired, delete the entry */
tdb_delete_bystring(cache, keystr);
/*
* We're expired, delete the entry. We can't use gencache_del
* here, because that uses gencache_get_data_blob for checking
* the existence of a record. We know the thing exists and
* directly store an empty value with 0 timeout.
*/
gencache_set(keystr, "", 0);
SAFE_FREE(databuf.dptr);
return False;
@ -218,6 +326,137 @@ bool gencache_get(const char *keystr, char **valstr, time_t *timeout)
return True;
}
struct stabilize_state {
bool written;
bool error;
};
static int stabilize_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA val,
void *priv);
/**
* Stabilize gencache
*
* Migrate the clear-if-first gencache data to the stable,
* transaction-based gencache.tdb
*/
bool gencache_stabilize(void)
{
struct stabilize_state state;
int res;
char *now;
if (!gencache_init()) {
return false;
}
res = tdb_transaction_start(cache);
if (res == -1) {
DEBUG(10, ("Could not start transaction on gencache.tdb: "
"%s\n", tdb_errorstr(cache)));
return false;
}
res = tdb_transaction_start(cache_notrans);
if (res == -1) {
tdb_transaction_cancel(cache);
DEBUG(10, ("Could not start transaction on "
"gencache_notrans.tdb: %s\n",
tdb_errorstr(cache_notrans)));
return false;
}
state.error = false;
state.written = false;
res = tdb_traverse(cache_notrans, stabilize_fn, &state);
if ((res == -1) || state.error) {
if ((tdb_transaction_cancel(cache_notrans) == -1)
|| (tdb_transaction_cancel(cache) == -1)) {
smb_panic("tdb_transaction_cancel failed\n");
}
return false;
}
if (!state.written) {
if ((tdb_transaction_cancel(cache_notrans) == -1)
|| (tdb_transaction_cancel(cache) == -1)) {
smb_panic("tdb_transaction_cancel failed\n");
}
return true;
}
res = tdb_transaction_commit(cache);
if (res == -1) {
DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
"%s\n", tdb_errorstr(cache)));
if (tdb_transaction_cancel(cache_notrans) == -1) {
smb_panic("tdb_transaction_cancel failed\n");
}
return false;
}
res = tdb_transaction_commit(cache_notrans);
if (res == -1) {
DEBUG(10, ("tdb_transaction_commit on gencache.tdb failed: "
"%s\n", tdb_errorstr(cache)));
return false;
}
now = talloc_asprintf(talloc_tos(), "%d", (int)time(NULL));
if (now != NULL) {
tdb_store(cache_notrans, last_stabilize_key(),
string_term_tdb_data(now), 0);
TALLOC_FREE(now);
}
return true;
}
static int stabilize_fn(struct tdb_context *tdb, TDB_DATA key, TDB_DATA val,
void *priv)
{
struct stabilize_state *state = (struct stabilize_state *)priv;
int res;
time_t timeout;
if (tdb_data_cmp(key, last_stabilize_key()) == 0) {
return 0;
}
if (!gencache_pull_timeout((char *)val.dptr, &timeout, NULL)) {
DEBUG(10, ("Ignoring invalid entry\n"));
return 0;
}
if ((timeout < time(NULL)) || (val.dsize == 0)) {
res = tdb_delete(cache, key);
if ((res == -1) && (tdb_error(cache) == TDB_ERR_NOEXIST)) {
res = 0;
} else {
state->written = true;
}
} else {
res = tdb_store(cache, key, val, 0);
if (res == 0) {
state->written = true;
}
}
if (res == -1) {
DEBUG(10, ("Transfer to gencache.tdb failed: %s\n",
tdb_errorstr(cache)));
state->error = true;
return -1;
}
if (tdb_delete(cache_notrans, key) == -1) {
DEBUG(10, ("tdb_delete from gencache_notrans.tdb failed: "
"%s\n", tdb_errorstr(cache_notrans)));
state->error = true;
return -1;
}
return 0;
}
/**
* Get existing entry from the cache file.
*
@ -387,6 +626,7 @@ struct gencache_iterate_state {
void *priv);
const char *pattern;
void *priv;
bool in_persistent;
};
static int gencache_iterate_fn(struct tdb_context *tdb, TDB_DATA key,
@ -402,6 +642,14 @@ static int gencache_iterate_fn(struct tdb_context *tdb, TDB_DATA key,
time_t timeout;
char *timeout_endp;
if (tdb_data_cmp(key, last_stabilize_key()) == 0) {
return 0;
}
if (state->in_persistent && tdb_exists(cache_notrans, key)) {
return 0;
}
if (key.dptr[key.dsize-1] == '\0') {
keystr = (char *)key.dptr;
} else {
@ -462,5 +710,10 @@ void gencache_iterate(void (*fn)(const char* key, const char *value, time_t time
state.fn = fn;
state.pattern = keystr_pattern;
state.priv = data;
state.in_persistent = false;
tdb_traverse(cache_notrans, gencache_iterate_fn, &state);
state.in_persistent = true;
tdb_traverse(cache, gencache_iterate_fn, &state);
}

2
source3/nmbd/nmbd.c
View File

@ -82,6 +82,8 @@ static void terminate(void)
/* If there was an async dns child - kill it. */
kill_async_dns_child();
gencache_stabilize();
pidfile_unlink();
exit(0);

1
source3/smbd/server.c
View File

@ -868,6 +868,7 @@ static void exit_server_common(enum server_exit_reason how,
if (am_parent) {
pidfile_unlink();
}
gencache_stabilize();
}
/* if we had any open SMB connections when we exited then we

2
source3/utils/net.c
View File

@ -762,6 +762,8 @@ static struct functable net_func[] = {
DEBUG(2,("return code = %d\n", rc));
gencache_stabilize();
libnetapi_free(c->netapi_ctx);
poptFreeContext(pc);

23
source3/utils/net_cache.c
View File

@ -297,6 +297,21 @@ static int net_cache_flush(struct net_context *c, int argc, const char **argv)
return 0;
}
static int net_cache_stabilize(struct net_context *c, int argc,
const char **argv)
{
if (c->display_usage) {
d_printf("Usage:\n"
"net cache flush\n"
" Delete all cache entries.\n");
return 0;
}
if (!gencache_stabilize()) {
return -1;
}
return 0;
}
/**
* Entry point to 'net cache' subfunctionality
*
@ -362,6 +377,14 @@ int net_cache(struct net_context *c, int argc, const char **argv)
"net cache flush\n"
" Delete all cache entries"
},
{
"stabilize",
net_cache_stabilize,
NET_TRANSPORT_LOCAL,
"Move transient cache content to stable storage",
"net cache stabilize\n"
" Move transient cache content to stable storage"
},
{NULL, NULL, 0, NULL, NULL}
};

2
source3/winbindd/winbindd.c
View File

@ -163,6 +163,8 @@ static void terminate(bool is_parent)
trustdom_cache_shutdown();
gencache_stabilize();
#if 0
if (interactive) {
TALLOC_CTX *mem_ctx = talloc_init("end_description");