network/ndisc: fix coalescing of ndisc routes when multiple router exists (#35119)

Fixes #33470.

src/network/networkd-json.c

@@ -164,45 +164,58 @@ static int nexthop_group_build_json(NextHop *nexthop, sd_json_variant **ret) {
         return 0;
 }
 
-static int nexthop_append_json(NextHop *n, sd_json_variant **array) {
-        _cleanup_(sd_json_variant_unrefp) sd_json_variant *group = NULL;
-        _cleanup_free_ char *flags = NULL, *protocol = NULL, *state = NULL;
+static int nexthop_append_json(NextHop *n, bool serializing, sd_json_variant **array) {
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *v = NULL;
         int r;
 
         assert(n);
         assert(array);
 
-        r = route_flags_to_string_alloc(n->flags, &flags);
-        if (r < 0)
-                return r;
-
-        r = route_protocol_to_string_alloc(n->protocol, &protocol);
-        if (r < 0)
-                return r;
-
-        r = network_config_state_to_string_alloc(n->state, &state);
-        if (r < 0)
-                return r;
-
-        r = nexthop_group_build_json(n, &group);
-        if (r < 0)
-                return r;
-
-        return sd_json_variant_append_arraybo(
-                        array,
+        r = sd_json_buildo(
+                        &v,
                         SD_JSON_BUILD_PAIR_UNSIGNED("ID", n->id),
-                        JSON_BUILD_PAIR_IN_ADDR_NON_NULL("Gateway", &n->gw.address, n->family),
-                        SD_JSON_BUILD_PAIR_UNSIGNED("Flags", n->flags),
-                        SD_JSON_BUILD_PAIR_STRING("FlagsString", strempty(flags)),
-                        SD_JSON_BUILD_PAIR_UNSIGNED("Protocol", n->protocol),
-                        SD_JSON_BUILD_PAIR_STRING("ProtocolString", protocol),
-                        SD_JSON_BUILD_PAIR_BOOLEAN("Blackhole", n->blackhole),
-                        JSON_BUILD_PAIR_VARIANT_NON_NULL("Group", group),
+                        SD_JSON_BUILD_PAIR_INTEGER("Family", n->family),
                         SD_JSON_BUILD_PAIR_STRING("ConfigSource", network_config_source_to_string(n->source)),
-                        SD_JSON_BUILD_PAIR_STRING("ConfigState", state));
+                        JSON_BUILD_PAIR_IN_ADDR_NON_NULL("ConfigProvider", &n->provider, n->family));
+        if (r < 0)
+                return r;
+
+        if (!serializing) {
+                _cleanup_(sd_json_variant_unrefp) sd_json_variant *group = NULL;
+                _cleanup_free_ char *flags = NULL, *protocol = NULL, *state = NULL;
+
+                r = route_flags_to_string_alloc(n->flags, &flags);
+                if (r < 0)
+                        return r;
+
+                r = route_protocol_to_string_alloc(n->protocol, &protocol);
+                if (r < 0)
+                        return r;
+
+                r = network_config_state_to_string_alloc(n->state, &state);
+                if (r < 0)
+                        return r;
+
+                r = nexthop_group_build_json(n, &group);
+                if (r < 0)
+                        return r;
+
+                r = sd_json_variant_merge_objectbo(
+                                &v,
+                                JSON_BUILD_PAIR_IN_ADDR_NON_NULL("Gateway", &n->gw.address, n->family),
+                                SD_JSON_BUILD_PAIR_UNSIGNED("Flags", n->flags),
+                                SD_JSON_BUILD_PAIR_STRING("FlagsString", strempty(flags)),
+                                SD_JSON_BUILD_PAIR_UNSIGNED("Protocol", n->protocol),
+                                SD_JSON_BUILD_PAIR_STRING("ProtocolString", protocol),
+                                SD_JSON_BUILD_PAIR_BOOLEAN("Blackhole", n->blackhole),
+                                JSON_BUILD_PAIR_VARIANT_NON_NULL("Group", group),
+                                SD_JSON_BUILD_PAIR_STRING("ConfigState", state));
+        }
+
+        return sd_json_variant_append_array(array, v);
 }
 
-static int nexthops_append_json(Manager *manager, int ifindex, sd_json_variant **v) {
+int nexthops_append_json(Manager *manager, int ifindex, sd_json_variant **v) {
         _cleanup_(sd_json_variant_unrefp) sd_json_variant *array = NULL;
         NextHop *nexthop;
         int r;
@@ -211,10 +224,21 @@ static int nexthops_append_json(Manager *manager, int ifindex, sd_json_variant *
         assert(v);
 
         HASHMAP_FOREACH(nexthop, manager->nexthops_by_id) {
-                if (nexthop->ifindex != ifindex)
-                        continue;
+                if (ifindex >= 0) {
+                        if (nexthop->ifindex != ifindex)
+                                continue;
+                } else {
+                        /* negative ifindex means we are serializing now. */
 
-                r = nexthop_append_json(nexthop, &array);
+                        if (nexthop->source == NETWORK_CONFIG_SOURCE_FOREIGN)
+                                continue;
+                        if (!nexthop_exists(nexthop))
+                                continue;
+
+                        log_nexthop_debug(nexthop, "Serializing", manager);
+                }
+
+                r = nexthop_append_json(nexthop, /* serializing = */ ifindex < 0, &array);
                 if (r < 0)
                         return r;
         }

src/network/networkd-json.h

@@ -9,6 +9,7 @@ typedef struct Link Link;
 typedef struct Manager Manager;
 
 int addresses_append_json(Link *link, bool serializing, sd_json_variant **v);
+int nexthops_append_json(Manager *manager, int ifindex, sd_json_variant **v);
 int routes_append_json(Manager *manager, int ifindex, sd_json_variant **v);
 int link_build_json(Link *link, sd_json_variant **ret);
 int manager_build_json(Manager *manager, sd_json_variant **ret);

src/network/networkd-ndisc.c

@@ -18,6 +18,7 @@
 #include "networkd-dhcp6.h"
 #include "networkd-manager.h"
 #include "networkd-ndisc.h"
+#include "networkd-nexthop.h"
 #include "networkd-queue.h"
 #include "networkd-route.h"
 #include "networkd-state-file.h"
@@ -149,6 +150,290 @@ static int ndisc_check_ready(Link *link) {
         return 0;
 }
 
+static int ndisc_remove_unused_nexthop(Link *link, NextHop *nexthop) {
+        int r;
+
+        assert(link);
+        assert(link->manager);
+        assert(link->ifindex > 0);
+        assert(nexthop);
+
+        if (nexthop->source != NETWORK_CONFIG_SOURCE_NDISC)
+                return 0;
+
+        if (nexthop->ifindex != link->ifindex)
+                return 0;
+
+        Route *route;
+        SET_FOREACH(route, nexthop->routes)
+                if (route_exists(route) || route_is_requesting(route))
+                        return 0;
+
+        Request *req;
+        ORDERED_SET_FOREACH(req, link->manager->request_queue) {
+                if (req->type != REQUEST_TYPE_ROUTE)
+                        continue;
+
+                route = ASSERT_PTR(req->userdata);
+                if (route->nexthop_id == nexthop->id)
+                        return 0;
+        }
+
+        r = nexthop_remove_and_cancel(nexthop, link->manager);
+        if (r < 0)
+                return log_link_debug_errno(link, r, "Failed to remove unused nexthop: %m");
+
+        return 0;
+}
+
+static int ndisc_remove_unused_nexthop_by_id(Link *link, uint32_t id) {
+        assert(link);
+        assert(link->manager);
+
+        if (id == 0)
+                return 0;
+
+        NextHop *nexthop;
+        if (nexthop_get_by_id(link->manager, id, &nexthop) < 0)
+                return 0;
+
+        return ndisc_remove_unused_nexthop(link, nexthop);
+}
+
+static int ndisc_remove_unused_nexthops(Link *link) {
+        int ret = 0;
+
+        assert(link);
+        assert(link->manager);
+
+        NextHop *nexthop;
+        HASHMAP_FOREACH(nexthop, link->manager->nexthops_by_id)
+                RET_GATHER(ret, ndisc_remove_unused_nexthop(link, nexthop));
+
+        return ret;
+}
+
+#define NDISC_NEXTHOP_APP_ID SD_ID128_MAKE(76,d2,0f,1f,76,1e,44,d1,97,3a,52,5c,05,68,b5,0d)
+
+static uint32_t ndisc_generate_nexthop_id(NextHop *nexthop, Link *link, sd_id128_t app_id, uint64_t trial) {
+        assert(nexthop);
+        assert(link);
+
+        struct siphash state;
+        siphash24_init(&state, app_id.bytes);
+        siphash24_compress_typesafe(nexthop->protocol, &state);
+        siphash24_compress_string(link->ifname, &state);
+        siphash24_compress_typesafe(nexthop->gw.address.in6, &state);
+        siphash24_compress_typesafe(nexthop->provider.in6, &state);
+        uint64_t n = htole64(trial);
+        siphash24_compress_typesafe(n, &state);
+
+        uint64_t result = htole64(siphash24_finalize(&state));
+        return (uint32_t) ((result & 0xffffffff) ^ (result >> 32));
+}
+
+static bool ndisc_nexthop_equal(NextHop *a, NextHop *b) {
+        assert(a);
+        assert(b);
+
+        if (a->source != b->source)
+                return false;
+        if (a->protocol != b->protocol)
+                return false;
+        if (a->ifindex != b->ifindex)
+                return false;
+        if (!in6_addr_equal(&a->provider.in6, &b->provider.in6))
+                return false;
+        if (!in6_addr_equal(&a->gw.address.in6, &b->gw.address.in6))
+                return false;
+
+        return true;
+}
+
+static bool ndisc_take_nexthop_id(NextHop *nexthop, NextHop *existing, Manager *manager) {
+        assert(nexthop);
+        assert(existing);
+        assert(manager);
+
+        if (!ndisc_nexthop_equal(nexthop, existing))
+                return false;
+
+        log_nexthop_debug(existing, "Found matching", manager);
+        nexthop->id = existing->id;
+        return true;
+}
+
+static int ndisc_nexthop_find_id(NextHop *nexthop, Link *link) {
+        NextHop *n;
+        Request *req;
+        int r;
+
+        assert(nexthop);
+        assert(link);
+        assert(link->manager);
+
+        sd_id128_t app_id;
+        r = sd_id128_get_machine_app_specific(NDISC_NEXTHOP_APP_ID, &app_id);
+        if (r < 0)
+                return r;
+
+        uint32_t id = ndisc_generate_nexthop_id(nexthop, link, app_id, 0);
+        if (nexthop_get_by_id(link->manager, id, &n) >= 0 &&
+            ndisc_take_nexthop_id(nexthop, n, link->manager))
+                return true;
+        if (nexthop_get_request_by_id(link->manager, id, &req) >= 0 &&
+            ndisc_take_nexthop_id(nexthop, req->userdata, link->manager))
+                return true;
+
+        HASHMAP_FOREACH(n, link->manager->nexthops_by_id)
+                if (ndisc_take_nexthop_id(nexthop, n, link->manager))
+                        return true;
+
+        ORDERED_SET_FOREACH(req, link->manager->request_queue) {
+                if (req->type != REQUEST_TYPE_NEXTHOP)
+                        continue;
+
+                if (ndisc_take_nexthop_id(nexthop, req->userdata, link->manager))
+                        return true;
+        }
+
+        return false;
+}
+
+static int ndisc_nexthop_new(Route *route, Link *link, NextHop **ret) {
+        _cleanup_(nexthop_unrefp) NextHop *nexthop = NULL;
+        int r;
+
+        assert(route);
+        assert(link);
+        assert(ret);
+
+        r = nexthop_new(&nexthop);
+        if (r < 0)
+                return r;
+
+        nexthop->source = NETWORK_CONFIG_SOURCE_NDISC;
+        nexthop->provider = route->provider;
+        nexthop->protocol = route->protocol == RTPROT_REDIRECT ? RTPROT_REDIRECT : RTPROT_RA;
+        nexthop->family = AF_INET6;
+        nexthop->gw.address = route->nexthop.gw;
+        nexthop->ifindex = link->ifindex;
+
+        r = ndisc_nexthop_find_id(nexthop, link);
+        if (r < 0)
+                return r;
+
+        *ret = TAKE_PTR(nexthop);
+        return 0;
+}
+
+static int ndisc_nexthop_acquire_id(NextHop *nexthop, Link *link) {
+        int r;
+
+        assert(nexthop);
+        assert(nexthop->id == 0);
+        assert(link);
+        assert(link->manager);
+
+        sd_id128_t app_id;
+        r = sd_id128_get_machine_app_specific(NDISC_NEXTHOP_APP_ID, &app_id);
+        if (r < 0)
+                return r;
+
+        for (uint64_t trial = 0; trial < 100; trial++) {
+                uint32_t id = ndisc_generate_nexthop_id(nexthop, link, app_id, trial);
+                if (id == 0)
+                        continue;
+
+                if (set_contains(link->manager->nexthop_ids, UINT32_TO_PTR(id)))
+                        continue; /* The ID is already used in a .network file. */
+
+                if (nexthop_get_by_id(link->manager, id, NULL) >= 0)
+                        continue; /* The ID is already used by an existing nexthop. */
+
+                if (nexthop_get_request_by_id(link->manager, id, NULL) >= 0)
+                        continue; /* The ID is already used by a nexthop being requested. */
+
+                log_link_debug(link, "Generated new ndisc nexthop ID for %s with trial %"PRIu64": %"PRIu32,
+                               IN6_ADDR_TO_STRING(&nexthop->gw.address.in6), trial, id);
+                nexthop->id = id;
+                return 0;
+        }
+
+        return log_link_debug_errno(link, SYNTHETIC_ERRNO(EBUSY), "Cannot find free nexthop ID for %s.",
+                                    IN6_ADDR_TO_STRING(&nexthop->gw.address.in6));
+}
+
+static int ndisc_nexthop_handler(sd_netlink *rtnl, sd_netlink_message *m, Request *req, Link *link, NextHop *nexthop) {
+        int r;
+
+        assert(link);
+
+        r = nexthop_configure_handler_internal(m, link, "Could not set NDisc route");
+        if (r <= 0)
+                return r;
+
+        r = ndisc_check_ready(link);
+        if (r < 0)
+                link_enter_failed(link);
+
+        return 1;
+}
+
+static int ndisc_request_nexthop(NextHop *nexthop, Link *link) {
+        int r;
+
+        assert(nexthop);
+        assert(link);
+
+        if (nexthop->id > 0)
+                return 0;
+
+        r = ndisc_nexthop_acquire_id(nexthop, link);
+        if (r < 0)
+                return r;
+
+        r = link_request_nexthop(link, nexthop, &link->ndisc_messages, ndisc_nexthop_handler);
+        if (r < 0)
+                return r;
+        if (r > 0)
+                link->ndisc_configured = false;
+
+        return 0;
+}
+
+static int ndisc_set_route_nexthop(Route *route, Link *link, bool request) {
+        _cleanup_(nexthop_unrefp) NextHop *nexthop = NULL;
+        int r;
+
+        assert(route);
+        assert(link);
+        assert(link->manager);
+
+        if (!link->manager->manage_foreign_nexthops)
+                goto finalize;
+
+        if (route->nexthop.family != AF_INET6 || in6_addr_is_null(&route->nexthop.gw.in6))
+                goto finalize;
+
+        r = ndisc_nexthop_new(route, link, &nexthop);
+        if (r < 0)
+                return r;
+
+        if (nexthop->id == 0 && !request)
+                goto finalize;
+
+        r = ndisc_request_nexthop(nexthop, link);
+        if (r < 0)
+                return r;
+
+        route->nexthop = (RouteNextHop) {};
+        route->nexthop_id = nexthop->id;
+
+finalize:
+        return route_adjust_nexthops(route, link);
+}
+
 static int ndisc_route_handler(sd_netlink *rtnl, sd_netlink_message *m, Request *req, Link *link, Route *route) {
         int r;
 
@@ -200,7 +485,7 @@ static int ndisc_request_route(Route *route, Link *link) {
         if (r < 0)
                 return r;
 
-        r = route_adjust_nexthops(route, link);
+        r = ndisc_set_route_nexthop(route, link, /* request = */ true);
         if (r < 0)
                 return r;
 
@@ -322,7 +607,7 @@ static int ndisc_remove_route(Route *route, Link *link) {
         assert(link);
         assert(link->manager);
 
-        r = route_adjust_nexthops(route, link);
+        r = ndisc_set_route_nexthop(route, link, /* request = */ false);
         if (r < 0)
                 return r;
 
@@ -362,7 +647,7 @@ static int ndisc_remove_route(Route *route, Link *link) {
                 }
         }
 
-        return ret;
+        return RET_GATHER(ret, ndisc_remove_unused_nexthop_by_id(link, route->nexthop_id));
 }
 
 static int ndisc_remove_router_route(Route *route, Link *link, sd_ndisc_router *rt) {
@@ -2112,6 +2397,8 @@ static int ndisc_drop_outdated(Link *link, const struct in6_addr *router, usec_t
                         RET_GATHER(ret, log_link_warning_errno(link, r, "Failed to remove outdated SLAAC route, ignoring: %m"));
         }
 
+        RET_GATHER(ret, ndisc_remove_unused_nexthops(link));
+
         SET_FOREACH(address, link->addresses) {
                 if (address->source != NETWORK_CONFIG_SOURCE_NDISC)
                         continue;
@@ -2772,6 +3059,8 @@ int link_drop_ndisc_config(Link *link, Network *network) {
                         if (r < 0)
                                 RET_GATHER(ret, log_link_warning_errno(link, r, "Failed to remove SLAAC route, ignoring: %m"));
                 }
+
+                RET_GATHER(ret, ndisc_remove_unused_nexthops(link));
         }
 
         /* If SLAAC address is disabled, drop all addresses. */

src/network/networkd-nexthop.c

@@ -105,7 +105,6 @@ static NextHop* nexthop_free(NextHop *nexthop) {
 }
 
 DEFINE_TRIVIAL_REF_UNREF_FUNC(NextHop, nexthop, nexthop_free);
-DEFINE_SECTION_CLEANUP_FUNCTIONS(NextHop, nexthop_unref);
 
 DEFINE_PRIVATE_HASH_OPS_WITH_VALUE_DESTRUCTOR(
                 nexthop_hash_ops,
@@ -123,7 +122,7 @@ DEFINE_PRIVATE_HASH_OPS_WITH_VALUE_DESTRUCTOR(
                 NextHop,
                 nexthop_detach);
 
-static int nexthop_new(NextHop **ret) {
+int nexthop_new(NextHop **ret) {
         _cleanup_(nexthop_unrefp) NextHop *nexthop = NULL;
 
         nexthop = new(NextHop, 1);
@@ -339,7 +338,7 @@ static int nexthop_get(Link *link, const NextHop *in, NextHop **ret) {
         return -ENOENT;
 }
 
-static int nexthop_get_request_by_id(Manager *manager, uint32_t id, Request **ret) {
+int nexthop_get_request_by_id(Manager *manager, uint32_t id, Request **ret) {
         Request *req;
 
         assert(manager);
@@ -459,7 +458,7 @@ static int nexthop_acquire_id(Manager *manager, NextHop *nexthop) {
         return -EBUSY;
 }
 
-static void log_nexthop_debug(const NextHop *nexthop, const char *str, Manager *manager) {
+void log_nexthop_debug(const NextHop *nexthop, const char *str, Manager *manager) {
         _cleanup_free_ char *state = NULL, *group = NULL, *flags = NULL;
         struct nexthop_grp *nhg;
         Link *link = NULL;
@@ -569,6 +568,33 @@ int nexthop_remove(NextHop *nexthop, Manager *manager) {
         return 0;
 }
 
+int nexthop_remove_and_cancel(NextHop *nexthop, Manager *manager) {
+        _cleanup_(request_unrefp) Request *req = NULL;
+        bool waiting = false;
+
+        assert(nexthop);
+        assert(nexthop->id > 0);
+        assert(manager);
+
+        /* If the nexthop is remembered by the manager, then use the remembered object. */
+        (void) nexthop_get_by_id(manager, nexthop->id, &nexthop);
+
+        /* Cancel the request for the nexthop. If the request is already called but we have not received the
+         * notification about the request, then explicitly remove the nexthop. */
+        if (nexthop_get_request_by_id(manager, nexthop->id, &req) >= 0) {
+                request_ref(req); /* avoid the request freed by request_detach() */
+                waiting = req->waiting_reply;
+                request_detach(req);
+                nexthop_cancel_requesting(nexthop);
+        }
+
+        /* If we know that the nexthop will come or already exists, remove it. */
+        if (waiting || (nexthop->manager && nexthop_exists(nexthop)))
+                return nexthop_remove(nexthop, manager);
+
+        return 0;
+}
+
 static int nexthop_configure(NextHop *nexthop, Link *link, Request *req) {
         _cleanup_(sd_netlink_message_unrefp) sd_netlink_message *m = NULL;
         int r;
@@ -633,19 +659,32 @@ static int nexthop_configure(NextHop *nexthop, Link *link, Request *req) {
         return request_call_netlink_async(link->manager->rtnl, m, req);
 }
 
-static int static_nexthop_handler(sd_netlink *rtnl, sd_netlink_message *m, Request *req, Link *link, NextHop *nexthop) {
+int nexthop_configure_handler_internal(sd_netlink_message *m, Link *link, const char *error_msg) {
         int r;
 
         assert(m);
         assert(link);
+        assert(error_msg);
 
         r = sd_netlink_message_get_errno(m);
         if (r < 0 && r != -EEXIST) {
-                log_link_message_warning_errno(link, m, r, "Could not set nexthop");
+                log_link_message_warning_errno(link, m, r, error_msg);
                 link_enter_failed(link);
-                return 1;
+                return 0;
         }
 
+        return 1;
+}
+
+static int static_nexthop_handler(sd_netlink *rtnl, sd_netlink_message *m, Request *req, Link *link, NextHop *nexthop) {
+        int r;
+
+        assert(link);
+
+        r = nexthop_configure_handler_internal(m, link, "Failed to set static nexthop");
+        if (r <= 0)
+                return r;
+
         if (link->static_nexthop_messages == 0) {
                 log_link_debug(link, "Nexthops set");
                 link->static_nexthops_configured = true;
@@ -740,7 +779,12 @@ static int nexthop_process_request(Request *req, Link *link, NextHop *nexthop) {
         return 1;
 }
 
-static int link_request_nexthop(Link *link, const NextHop *nexthop) {
+int link_request_nexthop(
+                Link *link,
+                const NextHop *nexthop,
+                unsigned *message_counter,
+                nexthop_netlink_handler_t netlink_handler) {
+
         _cleanup_(nexthop_unrefp) NextHop *tmp = NULL;
         NextHop *existing = NULL;
         int r;
@@ -780,8 +824,8 @@ static int link_request_nexthop(Link *link, const NextHop *nexthop) {
                                     nexthop_hash_func,
                                     nexthop_compare_func,
                                     nexthop_process_request,
-                                    &link->static_nexthop_messages,
-                                    static_nexthop_handler,
+                                    message_counter,
+                                    netlink_handler,
                                     NULL);
         if (r <= 0)
                 return r;
@@ -807,7 +851,7 @@ int link_request_static_nexthops(Link *link, bool only_ipv4) {
                 if (only_ipv4 && nh->family != AF_INET)
                         continue;
 
-                r = link_request_nexthop(link, nh);
+                r = link_request_nexthop(link, nh, &link->static_nexthop_messages, static_nexthop_handler);
                 if (r < 0)
                         return log_link_warning_errno(link, r, "Could not request nexthop: %m");
         }
@@ -866,16 +910,17 @@ int link_drop_nexthops(Link *link, bool only_static) {
                 if (!nexthop_exists(nexthop))
                         continue;
 
-                if (only_static) {
-                        if (nexthop->source != NETWORK_CONFIG_SOURCE_STATIC)
+                if (nexthop->source == NETWORK_CONFIG_SOURCE_FOREIGN) {
+                        if (only_static)
                                 continue;
-                } else {
+
                         /* Do not mark foreign nexthop when KeepConfiguration= is enabled. */
-                        if (nexthop->source == NETWORK_CONFIG_SOURCE_FOREIGN &&
-                            link->network &&
+                        if (link->network &&
                             FLAGS_SET(link->network->keep_configuration, KEEP_CONFIGURATION_STATIC))
                                 continue;
-                }
+
+                } else if (nexthop->source != NETWORK_CONFIG_SOURCE_STATIC)
+                        continue; /* Ignore dynamically configurad nexthops. */
 
                 /* Ignore nexthops bound to other links. */
                 if (nexthop->ifindex > 0 && nexthop->ifindex != link->ifindex)
@@ -1106,6 +1151,7 @@ int manager_rtnl_process_nexthop(sd_netlink *rtnl, sd_netlink_message *message,
                 NextHop *n = ASSERT_PTR(req->userdata);
 
                 nexthop->source = n->source;
+                nexthop->provider = n->provider;
         }
 
         r = sd_rtnl_message_get_family(message, &nexthop->family);

src/network/networkd-nexthop.h

@@ -16,13 +16,21 @@
 typedef struct Link Link;
 typedef struct Manager Manager;
 typedef struct Network Network;
+typedef struct NextHop NextHop;
+typedef int (*nexthop_netlink_handler_t)(
+                sd_netlink *rtnl,
+                sd_netlink_message *m,
+                Request *req,
+                Link *link,
+                NextHop *address);
 
-typedef struct NextHop {
+struct NextHop {
         Network *network;
         Manager *manager;
         ConfigSection *section;
         NetworkConfigSource source;
         NetworkConfigState state;
+        union in_addr_union provider; /* DHCP server or router address */
 
         unsigned n_ref;
 
@@ -44,12 +52,17 @@ typedef struct NextHop {
         /* For managing routes and nexthops that depend on this nexthop. */
         Set *nexthops;
         Set *routes;
-} NextHop;
+};
 
+void log_nexthop_debug(const NextHop *nexthop, const char *str, Manager *manager);
+
+int nexthop_new(NextHop **ret);
 NextHop* nexthop_ref(NextHop *nexthop);
 NextHop* nexthop_unref(NextHop *nexthop);
+DEFINE_SECTION_CLEANUP_FUNCTIONS(NextHop, nexthop_unref);
 
 int nexthop_remove(NextHop *nexthop, Manager *manager);
+int nexthop_remove_and_cancel(NextHop *nexthop, Manager *manager);
 
 int network_drop_invalid_nexthops(Network *network);
 
@@ -62,9 +75,16 @@ static inline int link_drop_static_nexthops(Link *link) {
 }
 void link_forget_nexthops(Link *link);
 
+int nexthop_configure_handler_internal(sd_netlink_message *m, Link *link, const char *error_msg);
+int link_request_nexthop(
+                Link *link,
+                const NextHop *nexthop,
+                unsigned *message_counter,
+                nexthop_netlink_handler_t netlink_handler);
 int link_request_static_nexthops(Link *link, bool only_ipv4);
 
 int nexthop_get_by_id(Manager *manager, uint32_t id, NextHop **ret);
+int nexthop_get_request_by_id(Manager *manager, uint32_t id, Request **ret);
 int nexthop_is_ready(Manager *manager, uint32_t id, NextHop **ret);
 int manager_rtnl_process_nexthop(sd_netlink *rtnl, sd_netlink_message *message, Manager *m);
 int manager_build_nexthop_ids(Manager *manager);

src/network/networkd-serialize.c

@@ -11,6 +11,7 @@
 #include "networkd-json.h"
 #include "networkd-link.h"
 #include "networkd-manager.h"
+#include "networkd-nexthop.h"
 #include "networkd-route.h"
 #include "networkd-serialize.h"
 
@@ -49,6 +50,10 @@ int manager_serialize(Manager *manager) {
         if (r < 0)
                 return r;
 
+        r = nexthops_append_json(manager, /* ifindex = */ -1, &v);
+        if (r < 0)
+                return r;
+
         r = routes_append_json(manager, /* ifindex = */ -1, &v);
         if (r < 0)
                 return r;
@@ -230,6 +235,63 @@ static int manager_deserialize_link(Manager *manager, sd_json_variant *v) {
         return ret;
 }
 
+typedef struct NextHopParam {
+        uint32_t id;
+        int family;
+        NetworkConfigSource source;
+        struct iovec provider;
+} NextHopParam;
+
+static void nexthop_param_done(NextHopParam *p) {
+        assert(p);
+
+        iovec_done(&p->provider);
+}
+
+static int manager_deserialize_nexthop(Manager *manager, sd_json_variant *v) {
+        static const sd_json_dispatch_field dispatch_table[] = {
+                { "ID",             _SD_JSON_VARIANT_TYPE_INVALID, sd_json_dispatch_uint32,             offsetof(NextHopParam, id),        SD_JSON_MANDATORY },
+                { "Family",         _SD_JSON_VARIANT_TYPE_INVALID, json_dispatch_address_family,        offsetof(NextHopParam, family),    SD_JSON_MANDATORY },
+                { "ConfigSource",   SD_JSON_VARIANT_STRING,        json_dispatch_network_config_source, offsetof(NextHopParam, source),    SD_JSON_MANDATORY },
+                { "ConfigProvider", SD_JSON_VARIANT_ARRAY,         json_dispatch_byte_array_iovec,      offsetof(NextHopParam, provider),  0                 },
+                {},
+        };
+
+        int r;
+
+        assert(manager);
+        assert(v);
+
+        _cleanup_(nexthop_param_done) NextHopParam p = {};
+        r = sd_json_dispatch(v, dispatch_table, SD_JSON_ALLOW_EXTENSIONS, &p);
+        if (r < 0)
+                return log_debug_errno(r, "Failed to dispatch nexthop from json variant: %m");
+
+        if (p.id == 0)
+                return log_debug_errno(SYNTHETIC_ERRNO(EINVAL), "Dispatched nexthop ID is zero.");
+
+        if (p.provider.iov_len != 0 && p.provider.iov_len != FAMILY_ADDRESS_SIZE(p.family))
+                return log_debug_errno(SYNTHETIC_ERRNO(EINVAL),
+                                       "Dispatched provider address size (%zu) is incompatible with the family (%s).",
+                                       p.provider.iov_len, af_to_ipv4_ipv6(p.family));
+
+        NextHop *nexthop;
+        r = nexthop_get_by_id(manager, p.id, &nexthop);
+        if (r < 0) {
+                log_debug_errno(r, "Cannot find deserialized nexthop (ID=%"PRIu32"): %m", p.id);
+                return 0; /* Already removed? */
+        }
+
+        if (nexthop->source != NETWORK_CONFIG_SOURCE_FOREIGN)
+                return 0; /* Huh?? Already deserialized?? */
+
+        nexthop->source = p.source;
+        memcpy_safe(&nexthop->provider, p.provider.iov_base, p.provider.iov_len);
+
+        log_nexthop_debug(nexthop, "Deserialized", manager);
+        return 0;
+}
+
 typedef struct RouteParam {
         Route route;
 
@@ -395,6 +457,9 @@ int manager_deserialize(Manager *manager) {
         JSON_VARIANT_ARRAY_FOREACH(i, sd_json_variant_by_key(v, "Interfaces"))
                 RET_GATHER(ret, manager_deserialize_link(manager, i));
 
+        JSON_VARIANT_ARRAY_FOREACH(i, sd_json_variant_by_key(v, "NextHops"))
+                RET_GATHER(ret, manager_deserialize_nexthop(manager, i));
+
         JSON_VARIANT_ARRAY_FOREACH(i, sd_json_variant_by_key(v, "Routes"))
                 RET_GATHER(ret, manager_deserialize_route(manager, i));
 

test/test-network/systemd-networkd-tests.py

@@ -6259,8 +6259,8 @@ class NetworkdRATests(unittest.TestCase, Utilities):
         check_output(f'{test_ndisc_send} --interface veth-peer --type redirect --target-address fe80::2 --redirect-destination 2002:da8:1:2:1a:2b:3c:4d')
         self.wait_route_dropped('veth99', '2002:da8:1:1:1a:2b:3c:4d proto redirect', ipv='-6', timeout_sec=10)
         self.wait_route_dropped('veth99', '2002:da8:1:2:1a:2b:3c:4d proto redirect', ipv='-6', timeout_sec=10)
-        self.wait_route('veth99', '2002:da8:1:1:1a:2b:3c:4d via fe80::1 proto redirect', ipv='-6', timeout_sec=10)
-        self.wait_route('veth99', '2002:da8:1:2:1a:2b:3c:4d via fe80::2 proto redirect', ipv='-6', timeout_sec=10)
+        self.wait_route('veth99', r'2002:da8:1:1:1a:2b:3c:4d nhid [0-9]* via fe80::1 proto redirect', ipv='-6', timeout_sec=10)
+        self.wait_route('veth99', r'2002:da8:1:2:1a:2b:3c:4d nhid [0-9]* via fe80::2 proto redirect', ipv='-6', timeout_sec=10)
 
         # Send Neighbor Advertisement without the router flag to announce the default router is not available anymore.
         # Then, verify that all redirect routes and the default route are dropped.
@@ -6412,14 +6412,14 @@ class NetworkdRATests(unittest.TestCase, Utilities):
 
         self.wait_address('client', '2002:da8:1:99:1034:56ff:fe78:9a00/64', ipv='-6', timeout_sec=10)
         self.wait_address('client', '2002:da8:1:98:1034:56ff:fe78:9a00/64', ipv='-6', timeout_sec=10)
-        self.wait_route('client', 'default via fe80::1034:56ff:fe78:9a99 proto ra metric 512', ipv='-6', timeout_sec=10)
-        self.wait_route('client', 'default via fe80::1034:56ff:fe78:9a98 proto ra metric 2048', ipv='-6', timeout_sec=10)
+        self.wait_route('client', r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a99 proto ra metric 512', ipv='-6', timeout_sec=10)
+        self.wait_route('client', r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a98 proto ra metric 2048', ipv='-6', timeout_sec=10)
 
         print('### ip -6 route show dev client default')
         output = check_output('ip -6 route show dev client default')
         print(output)
-        self.assertRegex(output, r'default via fe80::1034:56ff:fe78:9a99 proto ra metric 512 expires [0-9]*sec pref high')
-        self.assertRegex(output, r'default via fe80::1034:56ff:fe78:9a98 proto ra metric 2048 expires [0-9]*sec pref low')
+        self.assertRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a99 proto ra metric 512 expires [0-9]*sec pref high')
+        self.assertRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a98 proto ra metric 2048 expires [0-9]*sec pref low')
 
         with open(os.path.join(network_unit_dir, '25-veth-client.network'), mode='a', encoding='utf-8') as f:
             f.write('\n[Link]\nMACAddress=12:34:56:78:9a:01\n[IPv6AcceptRA]\nRouteMetric=100:200:300\n')
@@ -6429,14 +6429,14 @@ class NetworkdRATests(unittest.TestCase, Utilities):
 
         self.wait_address('client', '2002:da8:1:99:1034:56ff:fe78:9a01/64', ipv='-6', timeout_sec=10)
         self.wait_address('client', '2002:da8:1:98:1034:56ff:fe78:9a01/64', ipv='-6', timeout_sec=10)
-        self.wait_route('client', 'default via fe80::1034:56ff:fe78:9a99 proto ra metric 100', ipv='-6', timeout_sec=10)
-        self.wait_route('client', 'default via fe80::1034:56ff:fe78:9a98 proto ra metric 300', ipv='-6', timeout_sec=10)
+        self.wait_route('client', r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a99 proto ra metric 100', ipv='-6', timeout_sec=10)
+        self.wait_route('client', r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a98 proto ra metric 300', ipv='-6', timeout_sec=10)
 
         print('### ip -6 route show dev client default')
         output = check_output('ip -6 route show dev client default')
         print(output)
-        self.assertRegex(output, r'default via fe80::1034:56ff:fe78:9a99 proto ra metric 100 expires [0-9]*sec pref high')
-        self.assertRegex(output, r'default via fe80::1034:56ff:fe78:9a98 proto ra metric 300 expires [0-9]*sec pref low')
+        self.assertRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a99 proto ra metric 100 expires [0-9]*sec pref high')
+        self.assertRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a98 proto ra metric 300 expires [0-9]*sec pref low')
         self.assertNotIn('metric 512', output)
         self.assertNotIn('metric 2048', output)
 
@@ -6444,20 +6444,41 @@ class NetworkdRATests(unittest.TestCase, Utilities):
         remove_network_unit('25-veth-router-high.network', '25-veth-router-low.network')
         copy_network_unit('25-veth-router-high2.network', '25-veth-router-low2.network')
         networkctl_reload()
-        self.wait_route('client', 'default via fe80::1034:56ff:fe78:9a99 proto ra metric 300', ipv='-6', timeout_sec=10)
-        self.wait_route('client', 'default via fe80::1034:56ff:fe78:9a98 proto ra metric 100', ipv='-6', timeout_sec=10)
+        self.wait_route('client', r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a99 proto ra metric 300', ipv='-6', timeout_sec=10)
+        self.wait_route('client', r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a98 proto ra metric 100', ipv='-6', timeout_sec=10)
 
         print('### ip -6 route show dev client default')
         output = check_output('ip -6 route show dev client default')
         print(output)
-        self.assertRegex(output, r'default via fe80::1034:56ff:fe78:9a99 proto ra metric 300 expires [0-9]*sec pref low')
-        self.assertRegex(output, r'default via fe80::1034:56ff:fe78:9a98 proto ra metric 100 expires [0-9]*sec pref high')
-        self.assertNotRegex(output, 'default via fe80::1034:56ff:fe78:9a99 proto ra metric 100')
-        self.assertNotRegex(output, 'default via fe80::1034:56ff:fe78:9a98 proto ra metric 300')
+        self.assertRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a99 proto ra metric 300 expires [0-9]*sec pref low')
+        self.assertRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a98 proto ra metric 100 expires [0-9]*sec pref high')
+        self.assertNotRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a99 proto ra metric 100')
+        self.assertNotRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a98 proto ra metric 300')
         self.assertNotIn('metric 512', output)
         self.assertNotIn('metric 2048', output)
 
-    def test_ndisc_vs_static_route(self):
+        # Use the same preference, and check if the two routes are not coalesced. See issue #33470.
+        with open(os.path.join(network_unit_dir, '25-veth-router-high2.network'), mode='a', encoding='utf-8') as f:
+            f.write('\n[IPv6SendRA]\nRouterPreference=medium\n')
+        with open(os.path.join(network_unit_dir, '25-veth-router-low2.network'), mode='a', encoding='utf-8') as f:
+            f.write('\n[IPv6SendRA]\nRouterPreference=medium\n')
+        networkctl_reload()
+        self.wait_route('client', r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a99 proto ra metric 200', ipv='-6', timeout_sec=10)
+        self.wait_route('client', r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a98 proto ra metric 200', ipv='-6', timeout_sec=10)
+
+        print('### ip -6 route show dev client default')
+        output = check_output('ip -6 route show dev client default')
+        print(output)
+        self.assertRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a99 proto ra metric 200 expires [0-9]*sec pref medium')
+        self.assertRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9a98 proto ra metric 200 expires [0-9]*sec pref medium')
+        self.assertNotIn('pref high', output)
+        self.assertNotIn('pref low', output)
+        self.assertNotIn('metric 512', output)
+        self.assertNotIn('metric 2048', output)
+
+    def _test_ndisc_vs_static_route(self, manage_foreign_nexthops):
+        if not manage_foreign_nexthops:
+            copy_networkd_conf_dropin('networkd-manage-foreign-nexthops-no.conf')
         copy_network_unit('25-veth.netdev', '25-ipv6-prefix.network', '25-ipv6-prefix-veth-static-route.network')
         start_networkd()
         self.wait_online('veth99:routable', 'veth-peer:degraded')
@@ -6467,13 +6488,24 @@ class NetworkdRATests(unittest.TestCase, Utilities):
         output = check_output('ip -6 route show dev veth99 default')
         print(output)
         self.assertIn('via fe80::1034:56ff:fe78:9abd proto static metric 256 pref medium', output)
-        self.assertNotIn('proto ra', output)
+        if manage_foreign_nexthops:
+            self.assertRegex(output, r'default nhid [0-9]* via fe80::1034:56ff:fe78:9abd proto ra metric 256 expires [0-9]*sec pref medium')
+        else:
+            self.assertNotIn('proto ra', output)
+
+        print('### ip -6 nexthop show dev veth99')
+        output = check_output('ip -6 nexthop show dev veth99')
+        print(output)
+        if manage_foreign_nexthops:
+            self.assertRegex(output, r'id [0-9]* via fe80::1034:56ff:fe78:9abd dev veth99 scope link proto ra')
+        else:
+            self.assertEqual(output, '')
 
         # Also check if the static route is protected from RA with zero lifetime
         with open(os.path.join(network_unit_dir, '25-ipv6-prefix.network'), mode='a', encoding='utf-8') as f:
             f.write('\n[Network]\nIPv6SendRA=no\n')
         networkctl_reload() # This makes veth-peer being reconfigured, and send RA with zero lifetime
-        self.wait_route_dropped('veth99', 'default via fe80::1034:56ff:fe78:9abd proto ra metric 256', ipv='-6', timeout_sec=10)
+        self.wait_route_dropped('veth99', r'default (nhid [0-9]* |)via fe80::1034:56ff:fe78:9abd proto ra metric 256', ipv='-6', timeout_sec=10)
 
         print('### ip -6 route show dev veth99 default')
         output = check_output('ip -6 route show dev veth99 default')
@@ -6481,6 +6513,24 @@ class NetworkdRATests(unittest.TestCase, Utilities):
         self.assertIn('via fe80::1034:56ff:fe78:9abd proto static metric 256 pref medium', output)
         self.assertNotIn('proto ra', output)
 
+        # Check if nexthop is removed.
+        print('### ip -6 nexthop show dev veth99')
+        output = check_output('ip -6 nexthop show dev veth99')
+        print(output)
+        self.assertEqual(output, '')
+
+    def test_ndisc_vs_static_route(self):
+        first = True
+        for manage_foreign_nexthops in [True, False]:
+            if first:
+                first = False
+            else:
+                self.tearDown()
+
+            print(f'### test_ndisc_vs_static_route(manage_foreign_nexthops={manage_foreign_nexthops})')
+            with self.subTest(manage_foreign_nexthops=manage_foreign_nexthops):
+                self._test_ndisc_vs_static_route(manage_foreign_nexthops)
+
     # radvd supports captive portal since v2.20.
     # https://github.com/radvd-project/radvd/commit/791179a7f730decbddb2290ef0e34aa85d71b1bc
     @unittest.skipUnless(radvd_check_config('captive-portal.conf'), "Installed radvd doesn't support captive portals")
@@ -8382,10 +8432,15 @@ class NetworkdIPv6PrefixTests(unittest.TestCase, Utilities):
         self.assertIn('2001:db8:0:1::/64 proto ra', output)
         self.assertNotIn('2001:db8:0:2::/64 proto ra', output)
         self.assertNotIn('2001:db8:0:3::/64 proto ra', output)
-        self.assertRegex(output, '2001:db0:fff::/64 via fe80::1034:56ff:fe78:9abc')
+        self.assertRegex(output, r'2001:db0:fff::/64 nhid [0-9]* via fe80::1034:56ff:fe78:9abc')
         self.assertNotIn('2001:db1:fff::/64', output)
         self.assertNotIn('2001:db2:fff::/64', output)
 
+        print('### ip -6 nexthop show dev veth-peer')
+        output = check_output('ip -6 nexthop show dev veth-peer')
+        print(output)
+        self.assertRegex(output, r'id [0-9]* via fe80::1034:56ff:fe78:9abc dev veth-peer scope link proto ra')
+
         print('### ip -6 address show dev veth99')
         output = check_output('ip -6 address show dev veth99')
         print(output)
@@ -8434,9 +8489,14 @@ class NetworkdIPv6PrefixTests(unittest.TestCase, Utilities):
         print(output)
         self.assertIn('2001:db8:0:1::/64 proto ra', output)
         self.assertNotIn('2001:db8:0:2::/64 proto ra', output)
-        self.assertRegex(output, '2001:db0:fff::/64 via fe80::1034:56ff:fe78:9abc')
+        self.assertRegex(output, r'2001:db0:fff::/64 nhid [0-9]* via fe80::1034:56ff:fe78:9abc')
         self.assertNotIn('2001:db1:fff::/64', output)
 
+        print('### ip -6 nexthop show dev veth-peer')
+        output = check_output('ip -6 nexthop show dev veth-peer')
+        print(output)
+        self.assertRegex(output, r'id [0-9]* via fe80::1034:56ff:fe78:9abc dev veth-peer scope link proto ra')
+
         print('### ip -6 address show dev veth99')
         output = check_output('ip -6 address show dev veth99')
         print(output)