diff --git a/drivers/staging/greybus/core.c b/drivers/staging/greybus/core.c
index 7c701f398a4d..458020997294 100644
--- a/drivers/staging/greybus/core.c
+++ b/drivers/staging/greybus/core.c
@@ -198,15 +198,25 @@ struct greybus_host_device *greybus_create_hd(struct greybus_host_driver *driver
 	INIT_LIST_HEAD(&hd->connections);
 	ida_init(&hd->cport_id_map);
 
+	hd->endo = gb_endo_create(hd);
+	if (!hd->endo) {
+		greybus_remove_hd(hd);
+		return NULL;
+	}
+
 	return hd;
 }
 EXPORT_SYMBOL_GPL(greybus_create_hd);
 
 void greybus_remove_hd(struct greybus_host_device *hd)
 {
-	/* Tear down all modules that happen to be associated with this host
-	 * controller */
+	/*
+	 * Tear down all interfaces, modules, and the endo that is associated
+	 * with this host controller before freeing the memory associated with
+	 * the host controller.
+	 */
 	gb_remove_interfaces(hd);
+	gb_endo_remove(hd->endo);
 	kref_put_mutex(&hd->kref, free_hd, &hd_mutex);
 }
 EXPORT_SYMBOL_GPL(greybus_remove_hd);
diff --git a/drivers/staging/greybus/endo.c b/drivers/staging/greybus/endo.c
index e3bb25f7ec2b..28e1f2871bf8 100644
--- a/drivers/staging/greybus/endo.c
+++ b/drivers/staging/greybus/endo.c
@@ -91,7 +91,7 @@ static int create_modules(struct gb_endo *endo)
 	}
 
 	for (i = 0; endo_modules[i] != 0x00; ++i) {
-//		module = gb_module_create(&endo->dev, endo_modules[i]);
+		module = gb_module_create(&endo->dev, endo_modules[i]);
 		if (!module)
 			return -EINVAL;
 	}
@@ -99,16 +99,6 @@ static int create_modules(struct gb_endo *endo)
 	return 0;
 }
 
-static void remove_modules(struct gb_endo *endo)
-{
-	/*
-	 * We really don't care how many modules have been created, or what the
-	 * configuration of them are, let's just enumerate over everything in
-	 * the system and delete all found modules.
-	 */
-
-}
-
 struct gb_endo *gb_endo_create(struct greybus_host_device *hd)
 {
 	struct gb_endo *endo;
@@ -156,8 +146,8 @@ void gb_endo_remove(struct gb_endo *endo)
 	if (!endo)
 		return;
 
-	/* remove all modules first */
-	remove_modules(endo);
+	/* remove all modules for this endo */
+	gb_module_remove_all(endo);
 
 	device_unregister(&endo->dev);
 }
diff --git a/drivers/staging/greybus/greybus.h b/drivers/staging/greybus/greybus.h
index e0aae42fc4ce..109727f9f7c2 100644
--- a/drivers/staging/greybus/greybus.h
+++ b/drivers/staging/greybus/greybus.h
@@ -98,6 +98,8 @@ struct greybus_host_device {
 	/* Host device buffer constraints */
 	size_t buffer_size_max;
 
+	struct gb_endo *endo;
+
 	/* Private data for the host driver */
 	unsigned long hd_priv[0] __aligned(sizeof(s64));
 };
diff --git a/drivers/staging/greybus/interface.c b/drivers/staging/greybus/interface.c
index 5b1d5dde3c90..63665a2d8015 100644
--- a/drivers/staging/greybus/interface.c
+++ b/drivers/staging/greybus/interface.c
@@ -92,7 +92,7 @@ static struct gb_interface *gb_interface_create(struct greybus_host_device *hd,
 		return NULL;
 	}
 
-	module = gb_module_find_or_create(hd, get_module_id(interface_id));
+	module = gb_module_find(hd, get_module_id(interface_id));
 	if (!module)
 		return NULL;
 
@@ -157,7 +157,7 @@ static void gb_interface_destroy(struct gb_interface *intf)
 
 	module = intf->module;
 	device_unregister(&intf->dev);
-	gb_module_remove(module);
+	put_device(&module->dev);
 }
 
 /**
diff --git a/drivers/staging/greybus/module.c b/drivers/staging/greybus/module.c
index 780d163b3e00..202f141c7fe3 100644
--- a/drivers/staging/greybus/module.c
+++ b/drivers/staging/greybus/module.c
@@ -94,16 +94,22 @@ u8 get_module_id(u8 interface_id)
 	return interface_id;
 }
 
+struct module_find {
+	struct gb_endo *endo;
+	u8 module_id;
+};
+
 static int module_find(struct device *dev, void *data)
 {
 	struct gb_module *module;
-	u8 *module_id = data;
+	struct module_find *find = data;
 
 	if (!is_gb_module(dev))
 		return 0;
 
 	module = to_gb_module(dev);
-	if (module->module_id == *module_id)
+	if ((module->module_id == find->module_id) &&
+	    (module->dev.parent == &find->endo->dev))
 		return 1;
 
 	return 0;
@@ -113,21 +119,24 @@ static int module_find(struct device *dev, void *data)
  * Search the list of modules in the system.  If one is found, return it, with
  * the reference count incremented.
  */
-static struct gb_module *gb_module_find(u8 module_id)
+struct gb_module *gb_module_find(struct greybus_host_device *hd, u8 module_id)
 {
 	struct device *dev;
 	struct gb_module *module = NULL;
+	struct module_find find;
+
+	find.module_id = module_id;
+	find.endo = hd->endo;
 
 	dev = bus_find_device(&greybus_bus_type, NULL,
-			      &module_id, module_find);
+			      &find, module_find);
 	if (dev)
 		module = to_gb_module(dev);
 
 	return module;
 }
 
-static struct gb_module *gb_module_create(struct greybus_host_device *hd,
-					  u8 module_id)
+struct gb_module *gb_module_create(struct device *parent, u8 module_id)
 {
 	struct gb_module *module;
 	int retval;
@@ -137,12 +146,11 @@ static struct gb_module *gb_module_create(struct greybus_host_device *hd,
 		return NULL;
 
 	module->module_id = module_id;
-	module->refcount = 1;
-	module->dev.parent = hd->parent;
+	module->dev.parent = parent;
 	module->dev.bus = &greybus_bus_type;
 	module->dev.type = &greybus_module_type;
 	module->dev.groups = module_groups;
-	module->dev.dma_mask = hd->parent->dma_mask;
+	module->dev.dma_mask = parent->dma_mask;
 	device_initialize(&module->dev);
 	dev_set_name(&module->dev, "%d", module_id);
 
@@ -158,26 +166,22 @@ static struct gb_module *gb_module_create(struct greybus_host_device *hd,
 	return module;
 }
 
-struct gb_module *gb_module_find_or_create(struct greybus_host_device *hd,
-					   u8 module_id)
+static int module_remove(struct device *dev, void *data)
 {
 	struct gb_module *module;
+	struct gb_endo *endo = data;
 
-	module = gb_module_find(module_id);
-	if (module) {
-		module->refcount++;
-		return module;
-	}
+	if (!is_gb_module(dev))
+		return 0;
 
-	return gb_module_create(hd, module_id);
-}
-
-void gb_module_remove(struct gb_module *module)
-{
-	if (!module)
-		return;
-
-	if (!--module->refcount)
+	module = to_gb_module(dev);
+	if (module->dev.parent == &endo->dev)
 		device_unregister(&module->dev);
+
+	return 0;
 }
 
+void gb_module_remove_all(struct gb_endo *endo)
+{
+	bus_for_each_dev(&greybus_bus_type, NULL, endo, module_remove);
+}
diff --git a/drivers/staging/greybus/module.h b/drivers/staging/greybus/module.h
index f3e3bdd6a671..c23ac98fc1ba 100644
--- a/drivers/staging/greybus/module.h
+++ b/drivers/staging/greybus/module.h
@@ -13,16 +13,15 @@
 struct gb_module {
 	struct device dev;
 	u8 module_id;		/* Physical location within the Endo */
-	u16 refcount;
 };
 #define to_gb_module(d) container_of(d, struct gb_module, dev)
 
 struct greybus_host_device;
 
 /* Greybus "private" definitions */
-struct gb_module *gb_module_find_or_create(struct greybus_host_device *hd,
-					   u8 module_id);
-void gb_module_remove(struct gb_module *module);
+struct gb_module *gb_module_find(struct greybus_host_device *hd, u8 module_id);
+struct gb_module *gb_module_create(struct device *parent, u8 module_id);
+void gb_module_remove_all(struct gb_endo *endo);
 
 u8 get_module_id(u8 interface_id);