diff --git a/src/core/load-fragment.c b/src/core/load-fragment.c
index 00408c4b84..f7e82199f5 100644
--- a/src/core/load-fragment.c
+++ b/src/core/load-fragment.c
@@ -1860,7 +1860,7 @@ int config_parse_socket_service(
                 return -ENOEXEC;
         }
 
-        unit_ref_set(&s->service, x);
+        unit_ref_set(&s->service, UNIT(s), x);
 
         return 0;
 }
diff --git a/src/core/service.c b/src/core/service.c
index b6106593ee..e5113838bf 100644
--- a/src/core/service.c
+++ b/src/core/service.c
@@ -2684,7 +2684,7 @@ static int service_deserialize_item(Unit *u, const char *key, const char *value,
                 if (r < 0)
                         log_unit_debug_errno(u, r, "Failed to load accept-socket unit: %s", value);
                 else {
-                        unit_ref_set(&s->accept_socket, socket);
+                        unit_ref_set(&s->accept_socket, u, socket);
                         SOCKET(socket)->n_connections++;
                 }
 
@@ -3770,7 +3770,7 @@ int service_set_socket_fd(Service *s, int fd, Socket *sock, bool selinux_context
         s->socket_fd = fd;
         s->socket_fd_selinux_context_net = selinux_context_net;
 
-        unit_ref_set(&s->accept_socket, UNIT(sock));
+        unit_ref_set(&s->accept_socket, UNIT(s), UNIT(sock));
         return 0;
 }
 
diff --git a/src/core/slice.c b/src/core/slice.c
index ef2177279a..2228669e00 100644
--- a/src/core/slice.c
+++ b/src/core/slice.c
@@ -76,7 +76,7 @@ static int slice_add_parent_slice(Slice *s) {
         if (r < 0)
                 return r;
 
-        unit_ref_set(&u->slice, parent);
+        unit_ref_set(&u->slice, u, parent);
         return 0;
 }
 
diff --git a/src/core/socket.c b/src/core/socket.c
index f3e034d2af..cb42a3eb50 100644
--- a/src/core/socket.c
+++ b/src/core/socket.c
@@ -250,7 +250,7 @@ int socket_instantiate_service(Socket *s) {
         if (r < 0)
                 return r;
 
-        unit_ref_set(&s->service, u);
+        unit_ref_set(&s->service, UNIT(s), u);
 
         return unit_add_two_dependencies(UNIT(s), UNIT_BEFORE, UNIT_TRIGGERS, u, false, UNIT_DEPENDENCY_IMPLICIT);
 }
@@ -377,7 +377,7 @@ static int socket_add_extras(Socket *s) {
                         if (r < 0)
                                 return r;
 
-                        unit_ref_set(&s->service, x);
+                        unit_ref_set(&s->service, u, x);
                 }
 
                 r = unit_add_two_dependencies(u, UNIT_BEFORE, UNIT_TRIGGERS, UNIT_DEREF(s->service), true, UNIT_DEPENDENCY_IMPLICIT);
diff --git a/src/core/unit.c b/src/core/unit.c
index fd2cc6a667..761ce40f2a 100644
--- a/src/core/unit.c
+++ b/src/core/unit.c
@@ -362,7 +362,7 @@ bool unit_may_gc(Unit *u) {
         if (u->perpetual)
                 return false;
 
-        if (u->refs)
+        if (u->refs_by_target)
                 return false;
 
         if (sd_bus_track_count(u->bus_track) > 0)
@@ -662,9 +662,8 @@ void unit_free(Unit *u) {
         free(u->reboot_arg);
 
         unit_ref_unset(&u->slice);
-
-        while (u->refs)
-                unit_ref_unset(u->refs);
+        while (u->refs_by_target)
+                unit_ref_unset(u->refs_by_target);
 
         safe_close(u->ip_accounting_ingress_map_fd);
         safe_close(u->ip_accounting_egress_map_fd);
@@ -896,8 +895,8 @@ int unit_merge(Unit *u, Unit *other) {
                 return r;
 
         /* Redirect all references */
-        while (other->refs)
-                unit_ref_set(other->refs, u);
+        while (other->refs_by_target)
+                unit_ref_set(other->refs_by_target, other->refs_by_target->source, u);
 
         /* Merge dependencies */
         for (d = 0; d < _UNIT_DEPENDENCY_MAX; d++)
@@ -2976,8 +2975,7 @@ int unit_set_slice(Unit *u, Unit *slice) {
         if (UNIT_ISSET(u->slice) && u->cgroup_realized)
                 return -EBUSY;
 
-        unit_ref_unset(&u->slice);
-        unit_ref_set(&u->slice, slice);
+        unit_ref_set(&u->slice, u, slice);
         return 1;
 }
 
@@ -3986,30 +3984,32 @@ int unit_get_unit_file_preset(Unit *u) {
         return u->unit_file_preset;
 }
 
-Unit* unit_ref_set(UnitRef *ref, Unit *u) {
+Unit* unit_ref_set(UnitRef *ref, Unit *source, Unit *target) {
         assert(ref);
-        assert(u);
+        assert(source);
+        assert(target);
 
-        if (ref->unit)
+        if (ref->target)
                 unit_ref_unset(ref);
 
-        ref->unit = u;
-        LIST_PREPEND(refs, u->refs, ref);
-        return u;
+        ref->source = source;
+        ref->target = target;
+        LIST_PREPEND(refs_by_target, target->refs_by_target, ref);
+        return target;
 }
 
 void unit_ref_unset(UnitRef *ref) {
         assert(ref);
 
-        if (!ref->unit)
+        if (!ref->target)
                 return;
 
         /* We are about to drop a reference to the unit, make sure the garbage collection has a look at it as it might
          * be unreferenced now. */
-        unit_add_to_gc_queue(ref->unit);
+        unit_add_to_gc_queue(ref->target);
 
-        LIST_REMOVE(refs, ref->unit->refs, ref);
-        ref->unit = NULL;
+        LIST_REMOVE(refs_by_target, ref->target->refs_by_target, ref);
+        ref->source = ref->target = NULL;
 }
 
 static int user_from_unit_name(Unit *u, char **ret) {
diff --git a/src/core/unit.h b/src/core/unit.h
index 8e3d9bf323..286115cd87 100644
--- a/src/core/unit.h
+++ b/src/core/unit.h
@@ -123,8 +123,8 @@ struct UnitRef {
          * that we can merge two units if necessary and correct all
          * references to them */
 
-        Unit* unit;
-        LIST_FIELDS(UnitRef, refs);
+        Unit *source, *target;
+        LIST_FIELDS(UnitRef, refs_by_target);
 };
 
 typedef enum UnitCGroupBPFState {
@@ -187,7 +187,7 @@ struct Unit {
         char *job_timeout_reboot_arg;
 
         /* References to this */
-        LIST_HEAD(UnitRef, refs);
+        LIST_HEAD(UnitRef, refs_by_target);
 
         /* Conditions to check */
         LIST_HEAD(Condition, conditions);
@@ -724,11 +724,11 @@ void unit_trigger_notify(Unit *u);
 UnitFileState unit_get_unit_file_state(Unit *u);
 int unit_get_unit_file_preset(Unit *u);
 
-Unit* unit_ref_set(UnitRef *ref, Unit *u);
+Unit* unit_ref_set(UnitRef *ref, Unit *source, Unit *target);
 void unit_ref_unset(UnitRef *ref);
 
-#define UNIT_DEREF(ref) ((ref).unit)
-#define UNIT_ISSET(ref) (!!(ref).unit)
+#define UNIT_DEREF(ref) ((ref).target)
+#define UNIT_ISSET(ref) (!!(ref).target)
 
 int unit_patch_contexts(Unit *u);