1
0
mirror of https://github.com/samba-team/samba.git synced 2024-12-24 21:34:56 +03:00

This is the stat cache code - seems to work fine (needs heavy

NetBench testing though.... :-). Attempts to efficiently reduce
the number of stat() calls Samba does.
Jeremy.
This commit is contained in:
Jeremy Allison 0001-01-01 00:00:00 +00:00
parent cdc38c276d
commit d0e48a2d80
10 changed files with 327 additions and 51 deletions

View File

@ -419,6 +419,10 @@ union semun {
#define HAVE_NETGROUP 1
#endif
#if defined (HAVE_NETGROUP) && defined(HAVE_RPCSVC_YPCLNT_H)
#include "rpcsvc/ypclnt.h"
#endif
#ifndef ALLOW_CHANGE_PASSWORD
#if (defined(HAVE_TERMIOS_H) && defined(HAVE_DUP2) && defined(HAVE_SETSID))
#define ALLOW_CHANGE_PASSWORD 1

View File

@ -2000,7 +2000,9 @@ void sync_file(connection_struct *conn, files_struct *fsp);
BOOL fname_equal(char *name1, char *name2);
BOOL mangled_equal(char *name1, char *name2);
BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, BOOL *bad_path);
void print_stat_cache_statistics(void);
BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component,
BOOL *bad_path, struct stat *pst);
BOOL check_name(char *name,connection_struct *conn);
BOOL scan_directory(char *path, char *name,connection_struct *conn,BOOL docache);

View File

@ -908,6 +908,14 @@ struct bitmap {
#define IS_VETO_PATH(conn,path) ((conn) && is_in_path((path),(conn)->veto_list))
#define IS_VETO_OPLOCK_PATH(conn,path) ((conn) && is_in_path((path),(conn)->veto_oplock_list))
/*
* Used by the stat cache code to check if a returned
* stat structure is valid.
*/
#define VALID_STAT(st) (st.st_nlink != 0)
#define VALID_STAT_OF_DIR(st) (VALID_STAT(st) && S_ISDIR(st.st_mode))
#define SMBENCRYPT() (lp_encrypted_passwords())
/* the basic packet size, assuming no words or bytes */

View File

@ -29,10 +29,10 @@ extern fstring remote_machine;
extern BOOL use_mangled_map;
/****************************************************************************
check if two filenames are equal
this needs to be careful about whether we are case sensitive
Check if two filenames are equal.
This needs to be careful about whether we are case sensitive.
****************************************************************************/
BOOL fname_equal(char *name1, char *name2)
{
int l1 = strlen(name1);
@ -66,8 +66,9 @@ BOOL fname_equal(char *name1, char *name2)
/****************************************************************************
mangle the 2nd name and check if it is then equal to the first name
Mangle the 2nd name and check if it is then equal to the first name.
****************************************************************************/
BOOL mangled_equal(char *name1, char *name2)
{
pstring tmpname;
@ -81,6 +82,215 @@ BOOL mangled_equal(char *name1, char *name2)
return(strequal(name1,tmpname));
}
/****************************************************************************
Stat cache code used in unix_convert.
*****************************************************************************/
static int global_stat_cache_lookups;
static int global_stat_cache_misses;
static int global_stat_cache_hits;
/****************************************************************************
Stat cache statistics code.
*****************************************************************************/
void print_stat_cache_statistics(void)
{
double eff = ((double)global_stat_cache_lookups/100.0)*(double)global_stat_cache_hits;
DEBUG(0,("stat cache stats: lookups = %d, hits = %d, misses = %d, \
stat cache was %f%% effective.\n", global_stat_cache_lookups,
global_stat_cache_hits, global_stat_cache_misses, eff ));
}
typedef struct {
ubi_dlNode link;
int name_len;
pstring orig_name;
pstring translated_name;
} stat_cache_entry;
#define MAX_STAT_CACHE_SIZE 50
static ubi_dlList stat_cache = { NULL, (ubi_dlNodePtr)&stat_cache, 0};
/****************************************************************************
Compare two names in the stat cache.
*****************************************************************************/
static BOOL stat_name_equal( char *s1, char *s2)
{
return (case_sensitive ? (strcmp( s1, s2) == 0) : (StrCaseCmp(s1, s2) == 0));
}
/****************************************************************************
Compare two names in the stat cache.
*****************************************************************************/
static BOOL stat_name_equal_len( char *s1, char *s2, int len)
{
return (case_sensitive ? (strncmp( s1, s2, len) == 0) :
(StrnCaseCmp(s1, s2, len) == 0));
}
/****************************************************************************
Add an entry into the stat cache.
*****************************************************************************/
static void stat_cache_add( char *full_orig_name, char *orig_translated_path)
{
stat_cache_entry *scp;
pstring orig_name;
pstring translated_path;
int namelen = strlen(orig_translated_path);
/*
* Don't cache trivial valid directory entries.
*/
if(strequal(full_orig_name, ".") || strequal(full_orig_name, ".."))
return;
/*
* If we are in case insentive mode, we need to
* store names that need no translation - else, it
* would be a waste.
*/
if(case_sensitive && (strcmp(full_orig_name, orig_translated_path) == 0))
return;
/*
* Remove any trailing '/' characters from the
* translated path.
*/
pstrcpy(translated_path, orig_translated_path);
if(translated_path[namelen-1] == '/') {
translated_path[namelen-1] = '\0';
namelen--;
}
/*
* We will only replace namelen characters
* of full_orig_name.
* StrnCpy always null terminates.
*/
StrnCpy(orig_name, full_orig_name, namelen);
/*
* Check this name doesn't exist in the cache before we
* add it.
*/
for( scp = (stat_cache_entry *)ubi_dlFirst( &stat_cache); scp;
scp = (stat_cache_entry *)ubi_dlNext( scp )) {
if(stat_name_equal( scp->orig_name, orig_name) &&
(strcmp( scp->translated_name, translated_path) == 0)) {
/*
* Name does exist - promote it.
*/
if( (stat_cache_entry *)ubi_dlFirst( &stat_cache) != scp ) {
ubi_dlRemThis( &stat_cache, scp);
ubi_dlAddHead( &stat_cache, scp);
}
return;
}
}
if((scp = (stat_cache_entry *)malloc(sizeof(stat_cache_entry))) == NULL) {
DEBUG(0,("stat_cache_add: Out of memory !\n"));
return;
}
pstrcpy(scp->orig_name, orig_name);
pstrcpy(scp->translated_name, translated_path);
scp->name_len = namelen;
ubi_dlAddHead( &stat_cache, scp);
DEBUG(10,("stat_cache_add: Added entry %s -> %s\n", scp->orig_name, scp->translated_name ));
if(ubi_dlCount(&stat_cache) > MAX_STAT_CACHE_SIZE) {
scp = (stat_cache_entry *)ubi_dlRemTail( &stat_cache );
free((char *)scp);
return;
}
}
/****************************************************************************
Look through the stat cache for an entry - promote it to the top if found.
Return True if we translated (and did a scuccessful stat on) the entire name.
*****************************************************************************/
static BOOL stat_cache_lookup( char *name, char *dirpath, char **start, struct stat *pst)
{
stat_cache_entry *scp;
stat_cache_entry *longest_hit = NULL;
int namelen = strlen(name);
*start = name;
global_stat_cache_lookups++;
/*
* Don't lookup trivial valid directory entries.
*/
if(strequal(name, ".") || strequal(name, "..")) {
global_stat_cache_misses++;
return False;
}
for( scp = (stat_cache_entry *)ubi_dlFirst( &stat_cache); scp;
scp = (stat_cache_entry *)ubi_dlNext( scp )) {
if(scp->name_len <= namelen) {
if(stat_name_equal_len(scp->orig_name, name, scp->name_len)) {
if((longest_hit == NULL) || (longest_hit->name_len <= scp->name_len))
longest_hit = scp;
}
}
}
if(longest_hit == NULL) {
DEBUG(10,("stat_cache_lookup: cache miss on %s\n", name));
global_stat_cache_misses++;
return False;
}
global_stat_cache_hits++;
DEBUG(10,("stat_cache_lookup: cache hit for name %s. %s -> %s\n",
name, longest_hit->orig_name, longest_hit->translated_name ));
/*
* longest_hit is the longest match we got in the list.
* Check it exists - if so, overwrite the original name
* and then promote it to the top.
*/
if(sys_stat( longest_hit->translated_name, pst) != 0) {
/*
* Discard this entry.
*/
ubi_dlRemThis( &stat_cache, longest_hit);
free((char *)longest_hit);
return False;
}
memcpy(name, longest_hit->translated_name, longest_hit->name_len);
if( (stat_cache_entry *)ubi_dlFirst( &stat_cache) != longest_hit ) {
ubi_dlRemThis( &stat_cache, longest_hit);
ubi_dlAddHead( &stat_cache, longest_hit);
}
*start = &name[longest_hit->name_len];
if(**start == '/')
++*start;
StrnCpy( dirpath, longest_hit->translated_name, name - (*start));
return (namelen == longest_hit->name_len);
}
/****************************************************************************
This routine is called to convert names from the dos namespace to unix
namespace. It needs to handle any case conversions, mangling, format
@ -103,15 +313,21 @@ as Windows applications depend on ERRbadpath being returned if a component
of a pathname does not exist.
****************************************************************************/
BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component, BOOL *bad_path)
BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component,
BOOL *bad_path, struct stat *pst)
{
struct stat st;
char *start, *end;
char *start, *end, *orig_start;
pstring dirpath;
pstring orig_path;
int saved_errno;
BOOL component_was_mangled = False;
BOOL name_has_wildcard = False;
*dirpath = 0;
*bad_path = False;
if(pst)
memset( (char *)pst, '\0', sizeof(struct stat));
if(saved_last_component)
*saved_last_component = 0;
@ -166,16 +382,34 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component,
return(True);
}
start = name;
while (strncmp(start,"./",2) == 0)
start += 2;
pstrcpy(orig_path, name);
if(stat_cache_lookup( name, dirpath, &start, &st)) {
if(pst)
*pst = st;
return True;
}
/*
* stat the name - if it exists then we are all done!
*/
if (sys_stat(name,&st) == 0)
if (sys_stat(name,&st) == 0) {
stat_cache_add(orig_path, name);
DEBUG(5,("conversion finished %s -> %s\n",orig_path, name));
if(pst)
*pst = st;
return(True);
}
saved_errno = errno;
DEBUG(5,("unix_convert(%s)\n",name));
DEBUG(5,("unix_convert begin: name = %s, dirpath = %s, start = %s\n",
name, dirpath, start));
/*
* A special case - if we don't have any mangling chars and are case
@ -186,21 +420,20 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component,
!lp_strip_dot() && !use_mangled_map && (saved_errno != ENOENT))
return(False);
if(strchr(start,'?') || strchr(start,'*'))
name_has_wildcard = True;
/*
* Now we need to recursively match the name against the real
* directory structure.
*/
start = name;
while (strncmp(start,"./",2) == 0)
start += 2;
/*
* Match each part of the path name separately, trying the names
* as is first, then trying to scan the directory for matching names.
*/
for (;start;start = (end?end+1:(char *)NULL)) {
for (orig_start = start; start ; start = (end?end+1:(char *)NULL)) {
/*
* Pinpoint the end of this section of the filename.
*/
@ -231,6 +464,7 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component,
*end = '/';
return(False);
}
} else {
pstring rest;
@ -247,6 +481,7 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component,
/*
* Try to find this part of the path in the directory.
*/
if (strchr(start,'?') || strchr(start,'*') ||
!scan_directory(dirpath, start, conn, end?True:False)) {
if (end) {
@ -283,8 +518,10 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component,
* base of the filename.
*/
if (is_mangled(start))
if (is_mangled(start)) {
component_was_mangled = True;
check_mangled_cache( start );
}
DEBUG(5,("New file %s\n",start));
return(True);
@ -302,8 +539,18 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component,
/*
* Add to the dirpath that we have resolved so far.
*/
if (*dirpath) pstrcat(dirpath,"/");
pstrcat(dirpath,start);
if (*dirpath)
pstrcat(dirpath,"/");
pstrcat(dirpath,start);
/*
* Don't cache a name with mangled or wildcard components
* as this can change the size.
*/
if(!component_was_mangled && !name_has_wildcard)
stat_cache_add(orig_path, dirpath);
/*
* Restore the / that we wiped out earlier.
@ -312,10 +559,19 @@ BOOL unix_convert(char *name,connection_struct *conn,char *saved_last_component,
*end = '/';
}
/*
* Don't cache a name with mangled or wildcard components
* as this can change the size.
*/
if(!component_was_mangled && !name_has_wildcard)
stat_cache_add(orig_path, name);
/*
* The name has been resolved.
*/
DEBUG(5,("conversion finished %s\n",name));
DEBUG(5,("conversion finished %s -> %s\n",orig_path, name));
return(True);
}

View File

@ -490,7 +490,7 @@ int reply_ntcreate_and_X(connection_struct *conn,
set_posix_case_semantics(file_attributes);
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,NULL);
fsp = file_new();
if (!fsp) {
@ -743,7 +743,7 @@ static int call_nt_transact_create(connection_struct *conn,
set_posix_case_semantics(file_attributes);
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,NULL);
fsp = file_new();
if (!fsp) {

View File

@ -21,10 +21,6 @@
#include "includes.h"
#if (defined(HAVE_NETGROUP) && defined (WITH_AUTOMOUNT))
#include "rpcsvc/ypclnt.h"
#endif
extern int DEBUGLEVEL;
extern int Protocol;

View File

@ -723,6 +723,10 @@ void smbd_process(void)
DEBUG(0,("Reloading services after SIGHUP\n"));
reload_services(False);
reload_after_sighup = False;
/*
* Use this as an excuse to print some stats.
*/
print_stat_cache_statistics();
}
/* automatic timeout if all connections are closed */

View File

@ -751,14 +751,19 @@ int reply_chkpth(connection_struct *conn, char *inbuf,char *outbuf, int dum_size
pstring name;
BOOL ok = False;
BOOL bad_path = False;
struct stat st;
pstrcpy(name,smb_buf(inbuf) + 1);
unix_convert(name,conn,0,&bad_path);
unix_convert(name,conn,0,&bad_path,&st);
mode = SVAL(inbuf,smb_vwv0);
if (check_name(name,conn))
ok = directory_exist(name,NULL);
if (check_name(name,conn)) {
if(VALID_STAT(st))
ok = S_ISDIR(st.st_mode);
else
ok = directory_exist(name,NULL);
}
if (!ok)
{
@ -809,7 +814,7 @@ int reply_getatr(connection_struct *conn, char *inbuf,char *outbuf, int dum_size
BOOL bad_path = False;
pstrcpy(fname,smb_buf(inbuf) + 1);
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,&sbuf);
/* dos smetimes asks for a stat of "" - it returns a "hidden directory"
under WfWg - weird! */
@ -824,7 +829,7 @@ int reply_getatr(connection_struct *conn, char *inbuf,char *outbuf, int dum_size
else
if (check_name(fname,conn))
{
if (sys_stat(fname,&sbuf) == 0)
if (VALID_STAT(sbuf) || sys_stat(fname,&sbuf) == 0)
{
mode = dos_mode(conn,fname,&sbuf);
size = sbuf.st_size;
@ -881,15 +886,16 @@ int reply_setatr(connection_struct *conn, char *inbuf,char *outbuf, int dum_size
BOOL ok=False;
int mode;
time_t mtime;
struct stat st;
BOOL bad_path = False;
pstrcpy(fname,smb_buf(inbuf) + 1);
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,&st);
mode = SVAL(inbuf,smb_vwv0);
mtime = make_unix_date3(inbuf+smb_vwv1);
if (directory_exist(fname,NULL))
if (VALID_STAT_OF_DIR(st) || directory_exist(fname,NULL))
mode |= aDIR;
if (check_name(fname,conn))
ok = (dos_chmod(conn,fname,mode,NULL) == 0);
@ -990,7 +996,7 @@ int reply_search(connection_struct *conn, char *inbuf,char *outbuf, int dum_size
pstrcpy(directory,smb_buf(inbuf)+1);
pstrcpy(dir2,smb_buf(inbuf)+1);
unix_convert(directory,conn,0,&bad_path);
unix_convert(directory,conn,0,&bad_path,NULL);
unix_format(dir2);
if (!check_name(directory,conn))
@ -1250,7 +1256,7 @@ int reply_open(connection_struct *conn, char *inbuf,char *outbuf, int dum_size,
share_mode = SVAL(inbuf,smb_vwv0);
pstrcpy(fname,smb_buf(inbuf)+1);
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,NULL);
fsp = file_new();
if (!fsp)
@ -1351,7 +1357,7 @@ int reply_open_and_X(connection_struct *conn, char *inbuf,char *outbuf,int lengt
/* XXXX we need to handle passed times, sattr and flags */
pstrcpy(fname,smb_buf(inbuf));
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,NULL);
fsp = file_new();
if (!fsp)
@ -1485,7 +1491,7 @@ int reply_mknew(connection_struct *conn, char *inbuf,char *outbuf, int dum_size,
createmode = SVAL(inbuf,smb_vwv0);
pstrcpy(fname,smb_buf(inbuf)+1);
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,NULL);
if (createmode & aVOLID)
{
@ -1570,7 +1576,7 @@ int reply_ctemp(connection_struct *conn, char *inbuf,char *outbuf, int dum_size,
createmode = SVAL(inbuf,smb_vwv0);
pstrcpy(fname,smb_buf(inbuf)+1);
pstrcat(fname,"/TMXXXXXX");
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,NULL);
unixmode = unix_mode(conn,createmode);
@ -1674,7 +1680,7 @@ int reply_unlink(connection_struct *conn, char *inbuf,char *outbuf, int dum_size
DEBUG(3,("reply_unlink : %s\n",name));
unix_convert(name,conn,0,&bad_path);
unix_convert(name,conn,0,&bad_path,NULL);
p = strrchr(name,'/');
if (!p) {
@ -2778,7 +2784,7 @@ int reply_mkdir(connection_struct *conn, char *inbuf,char *outbuf, int dum_size,
BOOL bad_path = False;
pstrcpy(directory,smb_buf(inbuf) + 1);
unix_convert(directory,conn,0,&bad_path);
unix_convert(directory,conn,0,&bad_path,NULL);
if (check_name(directory, conn))
ret = sys_mkdir(directory,unix_mode(conn,aDIR));
@ -2872,7 +2878,7 @@ int reply_rmdir(connection_struct *conn, char *inbuf,char *outbuf, int dum_size,
BOOL bad_path = False;
pstrcpy(directory,smb_buf(inbuf) + 1);
unix_convert(directory,conn, NULL,&bad_path);
unix_convert(directory,conn, NULL,&bad_path,NULL);
if (check_name(directory,conn))
{
@ -3072,8 +3078,8 @@ int rename_internals(connection_struct *conn,
*directory = *mask = 0;
unix_convert(name,conn,0,&bad_path1);
unix_convert(newname,conn,newname_last_component,&bad_path2);
unix_convert(name,conn,0,&bad_path1,NULL);
unix_convert(newname,conn,newname_last_component,&bad_path2,NULL);
/*
* Split the old name into directory and last component
@ -3373,8 +3379,8 @@ int reply_copy(connection_struct *conn, char *inbuf,char *outbuf, int dum_size,
return(ERROR(ERRSRV,ERRinvdevice));
}
unix_convert(name,conn,0,&bad_path1);
unix_convert(newname,conn,0,&bad_path2);
unix_convert(name,conn,0,&bad_path1,NULL);
unix_convert(newname,conn,0,&bad_path2,NULL);
target_is_directory = directory_exist(newname,NULL);

View File

@ -287,7 +287,7 @@ max can be %d\n",
} /* end for num */
} /* end while 1 */
return True;
/* NOTREACHED return True; */
}
/****************************************************************************

View File

@ -214,7 +214,7 @@ static int call_trans2open(connection_struct *conn, char *inbuf, char *outbuf,
/* XXXX we need to handle passed times, sattr and flags */
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,NULL);
fsp = file_new();
if (!fsp)
@ -659,7 +659,7 @@ static int call_trans2findfirst(connection_struct *conn,
DEBUG(5,("path=%s\n",directory));
unix_convert(directory,conn,0,&bad_path);
unix_convert(directory,conn,0,&bad_path,NULL);
if(!check_name(directory,conn)) {
if((errno == ENOENT) && bad_path)
{
@ -1225,8 +1225,8 @@ static int call_trans2qfilepathinfo(connection_struct *conn,
info_level = SVAL(params,0);
fname = &fname1[0];
pstrcpy(fname,&params[6]);
unix_convert(fname,conn,0,&bad_path);
if (!check_name(fname,conn) || sys_stat(fname,&sbuf)) {
unix_convert(fname,conn,0,&bad_path,&sbuf);
if (!check_name(fname,conn) || (!VALID_STAT(sbuf) && sys_stat(fname,&sbuf))) {
DEBUG(3,("fileinfo of %s failed (%s)\n",fname,strerror(errno)));
if((errno == ENOENT) && bad_path)
{
@ -1459,7 +1459,7 @@ static int call_trans2setfilepathinfo(connection_struct *conn,
info_level = SVAL(params,0);
fname = fname1;
pstrcpy(fname,&params[6]);
unix_convert(fname,conn,0,&bad_path);
unix_convert(fname,conn,0,&bad_path,&st);
if(!check_name(fname, conn))
{
if((errno == ENOENT) && bad_path)
@ -1470,7 +1470,7 @@ static int call_trans2setfilepathinfo(connection_struct *conn,
return(UNIXERROR(ERRDOS,ERRbadpath));
}
if(sys_stat(fname,&st)!=0) {
if(!VALID_STAT(st) && sys_stat(fname,&st)!=0) {
DEBUG(3,("stat of %s failed (%s)\n", fname, strerror(errno)));
if((errno == ENOENT) && bad_path)
{
@ -1648,7 +1648,7 @@ static int call_trans2mkdir(connection_struct *conn,
DEBUG(3,("call_trans2mkdir : name = %s\n", directory));
unix_convert(directory,conn,0,&bad_path);
unix_convert(directory,conn,0,&bad_path,NULL);
if (check_name(directory,conn))
ret = sys_mkdir(directory,unix_mode(conn,aDIR));