features: add a directory-protection translator

This is useful to find all calls that remove a file from the protected
directory, including renames and internal calls.  Such calls will cause
a stack trace to be logged.  There's a filter script to add the needed
translators, and then the new functionality can be invoked with one of
the following commands.

	setfattr -n trusted.glusterfs.protect -v log $dir
	setfattr -n trusted.glusterfs.protect -v reject $dir
	setfattr -n trusted.glusterfs.protect -v anything_else $dir

The first logs calls, but still allows them.  The second rejects them
with EPERM.  The third turns off protection for that directory.

Change-Id: Iee4baaf8e837106be2b4099542cb7dcaae40428c
BUG: 888072
Signed-off-by: Jeff Darcy <jdarcy@redhat.com>
Reviewed-on: http://review.gluster.org/4496
Tested-by: Gluster Build System <jenkins@build.gluster.com>
Reviewed-by: Anand Avati <avati@redhat.com>

configure.ac

@@ -112,6 +112,8 @@ AC_CONFIG_FILES([Makefile
 		xlators/features/quiesce/src/Makefile
                 xlators/features/index/Makefile
                 xlators/features/index/src/Makefile
+		xlators/features/protect/Makefile
+		xlators/features/protect/src/Makefile
 		xlators/encryption/Makefile
 		xlators/encryption/rot-13/Makefile
 		xlators/encryption/rot-13/src/Makefile

extras/prot_filter.py

@@ -0,0 +1,144 @@
+#!/usr/bin/python
+
+"""
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+"""
+
+"""
+  INSTRUCTIONS
+  Put this in /usr/lib64/glusterfs/$version/filter to have it run automatically,
+  or else you'll have to run it by hand every time you change the volume
+  configuration.  Give it a list of volume names on which to enable the
+  protection functionality; it will deliberately ignore client volfiles for
+  other volumes, and all server volfiles.  It *will* include internal client
+  volfiles such as those used for NFS or rebalance/self-heal; this is a
+  deliberate choice so that it will catch deletions from those sources as well.
+"""
+
+volume_list = [ "jdtest" ]
+
+import copy
+import string
+import sys
+import types
+
+class Translator:
+    def __init__ (self, name):
+        self.name = name
+        self.xl_type = ""
+        self.opts = {}
+        self.subvols = []
+        self.dumped = False
+    def __repr__ (self):
+        return "<Translator %s>" % self.name
+
+def load (path):
+    # If it's a string, open it; otherwise, assume it's already a
+    # file-like object (most notably from urllib*).
+    if type(path) in types.StringTypes:
+        fp = file(path,"r")
+    else:
+        fp = path
+    all_xlators = {}
+    xlator = None
+    last_xlator = None
+    while True:
+        text = fp.readline()
+        if text == "":
+            break
+        text = text.split()
+        if not len(text):
+            continue
+        if text[0] == "volume":
+            if xlator:
+                raise RuntimeError, "nested volume definition"
+            xlator = Translator(text[1])
+            continue
+        if not xlator:
+            raise RuntimeError, "text outside volume definition"
+        if text[0] == "type":
+            xlator.xl_type = text[1]
+            continue
+        if text[0] == "option":
+            xlator.opts[text[1]] = string.join(text[2:])
+            continue
+        if text[0] == "subvolumes":
+            for sv in text[1:]:
+                xlator.subvols.append(all_xlators[sv])
+            continue
+        if text[0] == "end-volume":
+            all_xlators[xlator.name] = xlator
+            last_xlator = xlator
+            xlator = None
+            continue
+        raise RuntimeError, "unrecognized keyword %s" % text[0]
+    if xlator:
+        raise RuntimeError, "unclosed volume definition"
+    return all_xlators, last_xlator
+
+def generate (graph, last, stream=sys.stdout):
+    for sv in last.subvols:
+        if not sv.dumped:
+            generate(graph,sv,stream)
+            print >> stream, ""
+            sv.dumped = True
+    print >> stream, "volume %s" % last.name
+    print >> stream, "    type %s" % last.xl_type
+    for k, v in last.opts.iteritems():
+        print >> stream, "    option %s %s" % (k, v)
+    if last.subvols:
+        print >> stream, "    subvolumes %s" % string.join(
+            [ sv.name for sv in last.subvols ])
+    print >> stream, "end-volume"
+
+def push_filter (graph, old_xl, filt_type, opts={}):
+    new_type = "-" + filt_type.split("/")[1]
+    old_type = "-" + old_xl.xl_type.split("/")[1]
+    pos = old_xl.name.find(old_type)
+    if pos >= 0:
+        new_name = old_xl.name
+        old_name = new_name[:pos] + new_type + new_name[len(old_type)+pos:]
+    else:
+        new_name = old_xl.name + old_type
+        old_name = old_xl.name + new_type
+    new_xl = Translator(new_name)
+    new_xl.xl_type = old_xl.xl_type
+    new_xl.opts = old_xl.opts
+    new_xl.subvols = old_xl.subvols
+    graph[new_xl.name] = new_xl
+    old_xl.name = old_name
+    old_xl.xl_type = filt_type
+    old_xl.opts = opts
+    old_xl.subvols = [new_xl]
+    graph[old_xl.name] = old_xl
+
+if __name__ == "__main__":
+    path = sys.argv[1]
+    # Alow an override for debugging.
+    for extra in sys.argv[2:]:
+        volume_list.append(extra)
+    graph, last = load(path)
+    for v in volume_list:
+        if graph.has_key(v):
+            break
+    else:
+        print "No configured volumes found - aborting."
+        sys.exit(0)
+    for v in graph.values():
+        if v.xl_type == "cluster/distribute":
+            push_filter(graph,v,"features/prot_dht")
+        elif v.xl_type == "protocol/client":
+            push_filter(graph,v,"features/prot_client")
+    # We push debug/trace so that every fop gets a real frame, because DHT
+    # gets confused if STACK_WIND_TAIL causes certain fops to be invoked
+    # from anything other than a direct child.
+    for v in graph.values():
+        if v.xl_type == "features/prot_client":
+            push_filter(graph,v,"debug/trace")
+    generate(graph,last,stream=open(path,"w"))

xlators/debug/trace/src/trace.c

@@ -280,8 +280,8 @@ trace_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 LOG_ELEMENT (conf, string);
         }
 
-        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
 out:
+        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 

xlators/features/Makefile.am

@@ -1,3 +1,4 @@
-SUBDIRS = locks quota read-only mac-compat quiesce marker index # trash path-converter # filter
+SUBDIRS = locks quota read-only mac-compat quiesce marker index \
+	  protect # trash path-converter # filter
 
 CLEANFILES =

xlators/features/protect/Makefile.am

@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES = 

xlators/features/protect/src/Makefile.am

@@ -0,0 +1,21 @@
+xlator_LTLIBRARIES = prot_dht.la prot_client.la prot_server.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+prot_dht_la_LDFLAGS = -module -avoidversion
+prot_dht_la_SOURCES = prot_dht.c
+prot_dht_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+prot_client_la_LDFLAGS = -module -avoidversion
+prot_client_la_SOURCES = prot_client.c
+prot_client_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+prot_server_la_LDFLAGS = -module -avoidversion
+prot_server_la_SOURCES = prot_server.c
+prot_server_la_LIBADD  = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES = 
+

xlators/features/protect/src/prot_client.c

@@ -0,0 +1,215 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+#include <execinfo.h>
+
+#define NUM_FRAMES 20
+
+static char PROTECT_KEY[] = "trusted.glusterfs.protect";
+
+enum {
+        PROT_ACT_NONE = 0,
+        PROT_ACT_LOG,
+        PROT_ACT_REJECT,
+};
+
+void
+pcli_print_trace (char *name, call_frame_t *frame)
+{
+        void    *frames[NUM_FRAMES];
+        char    **symbols;
+        int     size;
+        int     i;
+
+        gf_log (name, GF_LOG_INFO, "Translator stack:");
+        while (frame) {
+                gf_log (name, GF_LOG_INFO, "%s (%s)",
+                        frame->wind_from, frame->this->name);
+                frame = frame->next;
+        }
+
+        size = backtrace(frames,NUM_FRAMES);
+        if (size <= 0) {
+                return;
+        }
+        symbols = backtrace_symbols(frames,size);
+        if (!symbols) {
+                return;
+        }
+
+        gf_log(name, GF_LOG_INFO, "Processor stack:");
+        for (i = 0; i < size; ++i) {
+                gf_log (name, GF_LOG_INFO, "%s", symbols[i]);
+        }
+        free(symbols);
+}
+
+int32_t
+pcli_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+        uint64_t        value;
+
+        if (newloc->parent == oldloc->parent) {
+                gf_log (this->name, GF_LOG_DEBUG, "rename in same directory");
+                goto simple_unwind;
+        }
+        if (!oldloc->parent) {
+                goto simple_unwind;
+        }
+        if (inode_ctx_get(oldloc->parent,this,&value) != 0) {
+                goto simple_unwind;
+        }
+
+        if (value != PROT_ACT_NONE) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "got rename for protected %s", oldloc->path);
+                pcli_print_trace(this->name,frame->next);
+                if (value == PROT_ACT_REJECT) {
+                        STACK_UNWIND_STRICT (rename, frame, -1, EPERM,
+                                             NULL, NULL, NULL, NULL, NULL,
+                                             xdata);
+                        return 0;
+                }
+        }
+
+simple_unwind:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->rename, oldloc, newloc,
+                         xdata);
+        return 0;
+}
+
+int32_t
+pcli_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+        data_t          *data;
+        uint64_t        value;
+
+        /*
+         * We can't use dict_get_str and strcmp here, because the value comes
+         * directly from the user and might not be NUL-terminated (it would
+         * be if we had set it ourselves.
+         */
+
+        data = dict_get(dict,PROTECT_KEY);
+        if (!data) {
+                goto simple_wind;
+        }
+
+        if (dict->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "attempted to mix %s with other keys", PROTECT_KEY);
+                goto simple_wind;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "got %s request", PROTECT_KEY);
+        if (!strncmp(data->data,"log",data->len)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "logging removals on %s", loc->path);
+                value = PROT_ACT_LOG;
+        }
+        else if (!strncmp(data->data,"reject",data->len)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "rejecting removals on %s", loc->path);
+                value = PROT_ACT_REJECT;
+        }
+        else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "removing protection on %s", loc->path);
+                value = PROT_ACT_NONE;
+        }
+        /* Right now the value doesn't matter - just the presence. */
+        if (inode_ctx_set(loc->inode,this,&value) != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set protection status for %s", loc->path);
+        }
+        STACK_UNWIND_STRICT (setxattr, frame, 0, 0, NULL);
+        return 0;
+
+simple_wind:
+        STACK_WIND_TAIL (frame,
+                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                         loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+pcli_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+             dict_t *xdata)
+{
+        uint64_t        value;
+
+        if (!loc->parent || (inode_ctx_get(loc->parent,this,&value) != 0)) {
+                goto simple_unwind;
+        }
+
+        if (value != PROT_ACT_NONE) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "got unlink for protected %s", loc->path);
+                pcli_print_trace(this->name,frame->next);
+                if (value == PROT_ACT_REJECT) {
+                        STACK_UNWIND_STRICT (unlink, frame, -1, EPERM,
+                                             NULL, NULL, NULL);
+                        return 0;
+                }
+        }
+
+simple_unwind:
+        STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                         FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+
+struct xlator_fops fops = {
+        .rename         = pcli_rename,
+        .setxattr       = pcli_setxattr,
+        .unlink         = pcli_unlink,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};

xlators/features/protect/src/prot_dht.c

@@ -0,0 +1,168 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+enum gf_pdht_mem_types_ {
+        gf_pdht_mt_coord_t = gf_common_mt_end + 1,
+        gf_pdht_mt_end
+};
+
+typedef struct {
+        pthread_mutex_t         lock;
+        uint16_t                refs;
+        int32_t                 op_ret;
+        int32_t                 op_errno;
+        dict_t                  *xdata;
+} pdht_coord_t;
+
+static char PROTECT_KEY[] = "trusted.glusterfs.protect";
+
+void
+pdht_unref_and_unlock (call_frame_t *frame, xlator_t *this,
+                       pdht_coord_t *coord)
+{
+        gf_boolean_t    should_unwind;
+
+        should_unwind = (--(coord->refs) == 0);
+        pthread_mutex_unlock(&coord->lock);
+
+        if (should_unwind) {
+                STACK_UNWIND_STRICT (setxattr, frame, 
+                                     coord->op_ret, coord->op_errno,
+                                     coord->xdata);
+                if (coord->xdata) {
+                        dict_unref(coord->xdata);
+                }
+                GF_FREE(coord);
+        }
+}
+
+int32_t
+pdht_recurse_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        pdht_coord_t    *coord = cookie;
+
+        pthread_mutex_lock(&coord->lock);
+        if (op_ret) {
+                coord->op_ret = op_ret;
+                coord->op_errno = op_errno;
+        }
+        if (xdata) {
+                if (coord->xdata) {
+                        dict_unref(coord->xdata);
+                }
+                coord->xdata = dict_ref(xdata);
+        }
+        pdht_unref_and_unlock(frame,this,coord);
+                
+        return 0;
+}
+
+void
+pdht_recurse (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+              int32_t flags, dict_t *xdata, xlator_t *xl, pdht_coord_t *coord)
+{
+        xlator_list_t   *iter;
+
+        if (!strcmp(xl->type,"features/prot_client")) {
+                pthread_mutex_lock(&coord->lock);
+                ++(coord->refs);
+                pthread_mutex_unlock(&coord->lock);
+                STACK_WIND_COOKIE (frame, pdht_recurse_cbk, coord, xl,
+                                   xl->fops->setxattr, loc, dict, flags, xdata);
+        }
+
+        else for (iter = xl->children; iter; iter = iter->next) {
+                pdht_recurse (frame, this, loc, dict, flags, xdata,
+                              iter->xlator, coord);
+        }
+}
+
+int32_t
+pdht_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+               int32_t flags, dict_t *xdata)
+{
+        pdht_coord_t    *coord;
+
+        if (!dict_get(dict,PROTECT_KEY)) {
+                goto simple_wind;
+        }
+
+        if (dict->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "attempted to mix %s with other keys", PROTECT_KEY);
+                goto simple_wind;
+        }
+
+        coord = GF_CALLOC(1,sizeof(*coord),gf_pdht_mt_coord_t);
+        if (!coord) {
+                gf_log (this->name, GF_LOG_WARNING, "allocation failed");
+                goto simple_wind;
+        }
+
+        pthread_mutex_init(&coord->lock,NULL);
+        coord->refs = 1;
+        coord->op_ret = 0;
+        coord->xdata = NULL;
+
+        pdht_recurse(frame,this,loc,dict,flags,xdata,this,coord);
+        pthread_mutex_lock(&coord->lock);
+        pdht_unref_and_unlock(frame,this,coord);
+
+        return 0;
+
+simple_wind:
+        STACK_WIND_TAIL (frame,
+                         FIRST_CHILD(this), FIRST_CHILD(this)->fops->setxattr,
+                         loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+struct xlator_fops fops = {
+        .setxattr = pdht_setxattr,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};

xlators/features/protect/src/prot_server.c

@@ -0,0 +1,51 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+
+int32_t
+init (xlator_t *this)
+{
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"translator not configured with exactly one child");
+		return -1;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+	return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+	return;
+}
+
+
+struct xlator_fops fops = {
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};