sin/samba-mirror
1
0
Fork 0
mirror of https://github.com/samba-team/samba.git synced 2025-02-03 13:47:25 +03:00
Code

s4-ldb: ldb indexing rewrite - part1

Browse Source 
This gets rid of the @IDXPTR approach to in-transaction indexing,
instead using an in-memory tdb to hold index values during a
transaction. This also cleans up a lot of the internal indexing logic,
hopefully making it easier to understand.

One of the big changes is in memory management, with a lot more use
made of talloc tricks to avoid copying dn lists, and shortcuts used to
avoid high intersection and union calculation costs.

The overall result is that a re-provision on my laptop goes from 48s
to a bit over 10s.
This commit is contained in:
Andrew Tridgell 2009-10-21 22:21:26 +11:00
parent c5de880c40
commit 859cf72692
4 changed files with 838 additions and 1008 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
source4/lib/ldb/ldb_tdb/ldb_cache.c
View File

@ -357,6 +357,8 @@ int ltdb_cache_load(struct ldb_module *module)
ltdb->cache->attributes == NULL) {
goto failed;
}
ltdb->cache->one_level_indexes = false;
ltdb->cache->attribute_indexes = false;
indexlist_dn = ldb_dn_new(module, ldb, LTDB_INDEXLIST);
if (indexlist_dn == NULL) goto failed;
@ -366,6 +368,13 @@ int ltdb_cache_load(struct ldb_module *module)
goto failed;
}
if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXONE) != NULL) {
ltdb->cache->one_level_indexes = true;
}
if (ldb_msg_find_element(ltdb->cache->indexlist, LTDB_IDXATTR) != NULL) {
ltdb->cache->attribute_indexes = true;
}
if (ltdb_attributes_load(module) == -1) {
goto failed;
}

1689
source4/lib/ldb/ldb_tdb/ldb_index.c
View File

File diff suppressed because it is too large Load Diff

130
source4/lib/ldb/ldb_tdb/ldb_tdb.c
View File

@ -250,11 +250,6 @@ int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flg
goto done;
}
ret = ltdb_index_add(module, msg);
if (ret != LDB_SUCCESS) {
tdb_delete(ltdb->tdb, tdb_key);
}
done:
talloc_free(tdb_key.dptr);
talloc_free(tdb_data.dptr);
@ -306,7 +301,7 @@ static int ltdb_add_internal(struct ldb_module *module,
return ret;
}
ret = ltdb_index_one(module, msg, 1);
ret = ltdb_index_add_new(module, msg);
if (ret != LDB_SUCCESS) {
return ret;
}
@ -340,7 +335,7 @@ static int ltdb_add(struct ltdb_context *ctx)
delete a record from the database, not updating indexes (used for deleting
index records)
*/
int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn)
static int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn)
{
void *data = ldb_module_get_private(module);
struct ltdb_private *ltdb = talloc_get_type(data, struct ltdb_private);
@ -385,14 +380,8 @@ static int ltdb_delete_internal(struct ldb_module *module, struct ldb_dn *dn)
goto done;
}
/* remove one level attribute */
ret = ltdb_index_one(module, msg, 0);
if (ret != LDB_SUCCESS) {
goto done;
}
/* remove any indexed attributes */
ret = ltdb_index_del(module, msg);
ret = ltdb_index_delete(module, msg);
if (ret != LDB_SUCCESS) {
goto done;
}
@ -453,9 +442,9 @@ static int find_element(const struct ldb_message *msg, const char *name)
returns 0 on success, -1 on failure (and sets errno)
*/
static int msg_add_element(struct ldb_context *ldb,
struct ldb_message *msg,
struct ldb_message_element *el)
static int ltdb_msg_add_element(struct ldb_context *ldb,
struct ldb_message *msg,
struct ldb_message_element *el)
{
struct ldb_message_element *e2;
unsigned int i;
@ -502,40 +491,35 @@ static int msg_delete_attribute(struct ldb_module *module,
struct ldb_message *msg, const char *name)
{
const char *dn;
unsigned int i, j;
unsigned int i;
int ret;
struct ldb_message_element *el;
dn = ldb_dn_get_linearized(msg->dn);
if (dn == NULL) {
return -1;
}
for (i=0;i<msg->num_elements;i++) {
if (ldb_attr_cmp(msg->elements[i].name, name) == 0) {
for (j=0;j<msg->elements[i].num_values;j++) {
ltdb_index_del_value(module, dn,
&msg->elements[i], j);
}
talloc_free(msg->elements[i].values);
if (msg->num_elements > (i+1)) {
memmove(&msg->elements[i],
&msg->elements[i+1],
sizeof(struct ldb_message_element)*
(msg->num_elements - (i+1)));
}
msg->num_elements--;
i--;
msg->elements = talloc_realloc(msg, msg->elements,
struct ldb_message_element,
msg->num_elements);
el = ldb_msg_find_element(msg, name);
if (el == NULL) {
return -1;
}
i = el - msg->elements;
/* per definition we find in a canonicalised message an
attribute only once. So we are finished here. */
return 0;
}
ret = ltdb_index_del_element(module, dn, el);
if (ret != LDB_SUCCESS) {
return ret;
}
/* Not found */
return -1;
talloc_free(el->values);
if (msg->num_elements > (i+1)) {
memmove(el, el+1, sizeof(*el) * (msg->num_elements - (i+1)));
}
msg->num_elements--;
msg->elements = talloc_realloc(msg, msg->elements,
struct ldb_message_element,
msg->num_elements);
return 0;
}
/*
@ -550,7 +534,7 @@ static int msg_delete_element(struct ldb_module *module,
{
struct ldb_context *ldb = ldb_module_get_ctx(module);
unsigned int i;
int found;
int found, ret;
struct ldb_message_element *el;
const struct ldb_schema_attribute *a;
@ -565,17 +549,22 @@ static int msg_delete_element(struct ldb_module *module,
for (i=0;i<el->num_values;i++) {
if (a->syntax->comparison_fn(ldb, ldb,
&el->values[i], val) == 0) {
&el->values[i], val) == 0) {
if (el->num_values == 1) {
return msg_delete_attribute(module, ldb, msg, name);
}
ret = ltdb_index_del_value(module, ldb_dn_get_linearized(msg->dn), el, i);
if (ret != LDB_SUCCESS) {
return -1;
}
if (i<el->num_values-1) {
memmove(&el->values[i], &el->values[i+1],
sizeof(el->values[i])*
(el->num_values-(i+1)));
}
el->num_values--;
if (el->num_values == 0) {
return msg_delete_attribute(module, ldb,
msg, name);
}
/* per definition we find in a canonicalised message an
attribute value only once. So we are finished here */
@ -669,10 +658,14 @@ int ltdb_modify_internal(struct ldb_module *module,
/* Checks if element already exists */
idx = find_element(msg2, el->name);
if (idx == -1) {
if (msg_add_element(ldb, msg2, el) != 0) {
if (ltdb_msg_add_element(ldb, msg2, el) != 0) {
ret = LDB_ERR_OTHER;
goto done;
}
ret = ltdb_index_add_element(module, msg->dn, el);
if (ret != LDB_SUCCESS) {
goto done;
}
} else {
/* We cannot add another value on a existing one
if the attribute is single-valued */
@ -703,8 +696,8 @@ int ltdb_modify_internal(struct ldb_module *module,
/* Now combine existing and new values to a new
attribute record */
vals = talloc_realloc(msg2->elements,
el2->values, struct ldb_val,
el2->num_values + el->num_values);
el2->values, struct ldb_val,
el2->num_values + el->num_values);
if (vals == NULL) {
ldb_oom(ldb);
ret = LDB_ERR_OTHER;
@ -718,6 +711,11 @@ int ltdb_modify_internal(struct ldb_module *module,
el2->values = vals;
el2->num_values += el->num_values;
ret = ltdb_index_add_element(module, msg->dn, el);
if (ret != LDB_SUCCESS) {
goto done;
}
}
break;
@ -740,15 +738,32 @@ int ltdb_modify_internal(struct ldb_module *module,
}
}
/* Delete the attribute if it exists in the DB */
msg_delete_attribute(module, ldb, msg2, el->name);
idx = find_element(msg2, el->name);
if (idx != -1) {
el2 = &(msg2->elements[idx]);
if (ldb_msg_element_compare(el, el2) == 0) {
/* we are replacing with the same values */
continue;
}
/* Delete the attribute if it exists in the DB */
ret = msg_delete_attribute(module, ldb, msg2, el->name);
if (ret != LDB_SUCCESS) {
goto done;
}
}
/* Recreate it with the new values */
if (msg_add_element(ldb, msg2, el) != 0) {
if (ltdb_msg_add_element(ldb, msg2, el) != 0) {
ret = LDB_ERR_OTHER;
goto done;
}
ret = ltdb_index_add_element(module, msg->dn, el);
if (ret != LDB_SUCCESS) {
goto done;
}
break;
case LDB_FLAG_MOD_DELETE:
@ -779,15 +794,8 @@ int ltdb_modify_internal(struct ldb_module *module,
ret = LDB_ERR_NO_SUCH_ATTRIBUTE;
goto done;
}
ret = ltdb_index_del_value(module, dn,
&msg->elements[i], j);
if (ret != LDB_SUCCESS) {
goto done;
}
}
}
break;
default:
ldb_asprintf_errstring(ldb,

18
source4/lib/ldb/ldb_tdb/ldb_tdb.h
View File

@ -17,6 +17,8 @@ struct ltdb_private {
struct ltdb_cache {
struct ldb_message *indexlist;
struct ldb_message *attributes;
bool one_level_indexes;
bool attribute_indexes;
struct {
char *name;
@ -58,7 +60,7 @@ struct ltdb_context {
#define LTDB_INDEX "@INDEX"
#define LTDB_INDEXLIST "@INDEXLIST"
#define LTDB_IDX "@IDX"
#define LTDB_IDXPTR "@IDXPTR"
#define LTDB_IDXVERSION "@IDXVERSION"
#define LTDB_IDXATTR "@IDXATTR"
#define LTDB_IDXONE "@IDXONE"
#define LTDB_BASEINFO "@BASEINFO"
@ -83,9 +85,13 @@ int ltdb_check_at_attributes_values(const struct ldb_val *value);
struct ldb_parse_tree;
int ltdb_search_indexed(struct ltdb_context *ctx, uint32_t *);
int ltdb_index_add(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_index_del(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_index_one(struct ldb_module *module, const struct ldb_message *msg, int add);
int ltdb_index_add_new(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_index_delete(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_index_del_element(struct ldb_module *module, const char *dn, struct ldb_message_element *el);
int ltdb_index_add_element(struct ldb_module *module, struct ldb_dn *dn,
struct ldb_message_element *el);
int ltdb_index_del_value(struct ldb_module *module, const char *dn,
struct ldb_message_element *el, int v_idx);
int ltdb_reindex(struct ldb_module *module);
int ltdb_index_transaction_start(struct ldb_module *module);
int ltdb_index_transaction_commit(struct ldb_module *module);
@ -122,12 +128,8 @@ int ltdb_lock_read(struct ldb_module *module);
int ltdb_unlock_read(struct ldb_module *module);
struct TDB_DATA ltdb_key(struct ldb_module *module, struct ldb_dn *dn);
int ltdb_store(struct ldb_module *module, const struct ldb_message *msg, int flgs);
int ltdb_delete_noindex(struct ldb_module *module, struct ldb_dn *dn);
int ltdb_modify_internal(struct ldb_module *module, const struct ldb_message *msg);
int ltdb_index_del_value(struct ldb_module *module, const char *dn,
struct ldb_message_element *el, int v_idx);
struct tdb_context *ltdb_wrap_open(TALLOC_CTX *mem_ctx,
const char *path, int hash_size, int tdb_flags,
int open_flags, mode_t mode,