diff --git a/frr/zebra_dplane_grout.c b/frr/zebra_dplane_grout.c index 5170bea7d..2f5cbd4d5 100644 --- a/frr/zebra_dplane_grout.c +++ b/frr/zebra_dplane_grout.c @@ -710,6 +710,16 @@ static void dplane_grout_connect(struct event *) { &grout_ctx.dg_t_dplane_update ); + // redone on every connection: a restarted grout comes back with its defaults + struct gr_iface_config_set_req req = { + .flush_routes_on_down = true, + .skip_route_events_on_iface_down = true, + .set_attrs = GR_IFACE_CONFIG_SET_FLUSH_ROUTES | GR_IFACE_CONFIG_SET_SKIP_EVENTS, + }; + if (gr_api_client_send_recv(grout_ctx.client, GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL) + < 0) + gr_log_err("GR_IFACE_CONFIG_SET: %s", strerror(errno)); + gr_log_notice("connected, monitoring iface/ip events"); } diff --git a/modules/dhcp/control/client.c b/modules/dhcp/control/client.c index 6dc94dec3..7f76389f7 100644 --- a/modules/dhcp/control/client.c +++ b/modules/dhcp/control/client.c @@ -198,7 +198,7 @@ static void dhcp_expire_callback(evutil_socket_t, short, void *arg) { if (client->offered_ip != 0 && client->prefixlen != 0) addr4_delete(iface->id, client->offered_ip, client->prefixlen); if (client->router_ip != 0) - rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3); + rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true); client->state = DHCP_STATE_INIT; client->offered_ip = 0; @@ -370,7 +370,7 @@ void dhcp_input_cb(void *obj, uintptr_t, const struct control_queue_drain *drain if (client->offered_ip != 0 && client->prefixlen != 0) addr4_delete(iface->id, client->offered_ip, client->prefixlen); if (client->router_ip != 0) - rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3); + rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true); client->state = DHCP_STATE_INIT; client->offered_ip = 0; @@ -454,7 +454,7 @@ static int dhcp_stop(uint16_t iface_id) { if (client->offered_ip != 0 && client->prefixlen != 0) addr4_delete(iface->id, client->offered_ip, client->prefixlen); if (client->router_ip != 0) - rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3); + rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true); dhcp_cancel_timers(client); diff --git a/modules/infra/api/gr_infra.h b/modules/infra/api/gr_infra.h index 72c1f79e7..45f2c66dd 100644 --- a/modules/infra/api/gr_infra.h +++ b/modules/infra/api/gr_infra.h @@ -218,6 +218,14 @@ struct gr_port_rxq_map { uint16_t enabled; }; +// Global interface behaviour configuration. +struct gr_iface_config { + // Delete the routes of an interface when it goes administratively down. + bool flush_routes_on_down; + // Do not emit route events for the routes deleted with an interface. + bool skip_route_events_on_iface_down; +}; + // Infrastructure statistics entry. struct gr_stat { char name[64]; @@ -252,6 +260,8 @@ enum gr_infra_requests : uint32_t { GR_IFACE_MAC_DEL, GR_IFACE_MAC_LIST, GR_IFACE_MAC_SET, + GR_IFACE_CONFIG_GET, + GR_IFACE_CONFIG_SET, }; enum gr_infra_events : uint32_t { @@ -363,6 +373,23 @@ struct gr_iface_set_req { GR_REQ(GR_IFACE_SET, struct gr_iface_set_req, struct gr_empty); +// Interface configuration attributes flags. +#define GR_IFACE_CONFIG_SET_FLUSH_ROUTES GR_BIT64(0) +#define GR_IFACE_CONFIG_SET_SKIP_EVENTS GR_BIT64(1) + +struct gr_iface_config_set_req { + BASE(gr_iface_config); + uint64_t set_attrs; // Bit mask of GR_IFACE_CONFIG_SET_*. +}; + +GR_REQ(GR_IFACE_CONFIG_SET, struct gr_iface_config_set_req, struct gr_empty); + +struct gr_iface_config_get_resp { + BASE(gr_iface_config); +}; + +GR_REQ(GR_IFACE_CONFIG_GET, struct gr_empty, struct gr_iface_config_get_resp); + // Get interface statistics. struct gr_iface_stats { uint16_t iface_id; diff --git a/modules/infra/api/iface.c b/modules/infra/api/iface.c index abe552652..029e95999 100644 --- a/modules/infra/api/iface.c +++ b/modules/infra/api/iface.c @@ -323,6 +323,31 @@ static void iface_metrics_collect(struct metrics_writer *w) { } } +static struct gr_iface_config iface_config; + +const struct gr_iface_config *iface_config_get(void) { + return &iface_config; +} + +static struct api_out config_get(const void *, struct api_ctx *) { + struct gr_iface_config_get_resp *resp = calloc(1, sizeof(*resp)); + if (resp == NULL) + return api_out(ENOMEM, 0, NULL); + resp->base = iface_config; + return api_out(0, sizeof(*resp), resp); +} + +static struct api_out config_set(const void *request, struct api_ctx *) { + const struct gr_iface_config_set_req *req = request; + + if (req->set_attrs & GR_IFACE_CONFIG_SET_FLUSH_ROUTES) + iface_config.flush_routes_on_down = req->flush_routes_on_down; + if (req->set_attrs & GR_IFACE_CONFIG_SET_SKIP_EVENTS) + iface_config.skip_route_events_on_iface_down = req->skip_route_events_on_iface_down; + + return api_out(0, 0, NULL); +} + static struct metrics_collector iface_collector = { .name = "iface", .collect = iface_metrics_collect, @@ -338,6 +363,8 @@ RTE_INIT(infra_api_init) { api_handler(GR_IFACE_MAC_LIST, iface_mac_list); api_handler(GR_IFACE_MAC_SET, iface_mac_set); api_handler(GR_IFACE_SET, iface_set); + api_handler(GR_IFACE_CONFIG_GET, config_get); + api_handler(GR_IFACE_CONFIG_SET, config_set); event_serializer(GR_EVENT_IFACE_ADD, iface_event_serialize); event_serializer(GR_EVENT_IFACE_POST_ADD, iface_event_serialize); event_serializer(GR_EVENT_IFACE_PRE_REMOVE, iface_event_serialize); diff --git a/modules/infra/api/nexthop.c b/modules/infra/api/nexthop.c index df6b3e599..78c0678ca 100644 --- a/modules/infra/api/nexthop.c +++ b/modules/infra/api/nexthop.c @@ -67,7 +67,7 @@ static struct api_out nh_del(const void *request, struct api_ctx *) { return api_out(EBUSY, 0, NULL); } - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); // The nexthop *may* still have one ref_count when it has been created // manually from the API (see nh_add()). Implicit nexthops created when // creating a gateway route will not have that extra ref_count. @@ -142,7 +142,7 @@ static struct api_out nh_flush(const void *request, struct api_ctx *) { if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS) continue; } - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); while (nh->ref_count > 0) nexthop_decref(nh); } diff --git a/modules/infra/cli/route.c b/modules/infra/cli/route.c index 8947ee940..39989a64e 100644 --- a/modules/infra/cli/route.c +++ b/modules/infra/cli/route.c @@ -97,6 +97,28 @@ static cmd_status_t route_list(struct gr_api_client *c, const struct ec_pnode *p return ret < 0 ? CMD_ERROR : CMD_SUCCESS; } +static cmd_status_t route_flush_config_set(struct gr_api_client *c, const struct ec_pnode *p) { + struct gr_iface_config_set_req req = {0}; + const char *flush, *skip; + + flush = arg_str(p, "FLUSH"); + if (flush != NULL) { + req.flush_routes_on_down = strcmp(flush, "on") == 0; + req.set_attrs |= GR_IFACE_CONFIG_SET_FLUSH_ROUTES; + } + + skip = arg_str(p, "SKIP"); + if (skip != NULL) { + req.skip_route_events_on_iface_down = strcmp(skip, "on") == 0; + req.set_attrs |= GR_IFACE_CONFIG_SET_SKIP_EVENTS; + } + + if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL) < 0) + return CMD_ERROR; + + return CMD_SUCCESS; +} + static cmd_status_t route_config_set(struct gr_api_client *c, const struct ec_pnode *p) { struct cli_route_ops *ops; cmd_status_t ret; @@ -140,6 +162,19 @@ static cmd_status_t route_config_show(struct gr_api_client *c, const struct ec_p gr_table_free(table); + if (ret == 0) { + const struct gr_iface_config_get_resp *resp; + void *resp_ptr = NULL; + + if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_GET, 0, NULL, &resp_ptr) < 0) + return CMD_ERROR; + resp = resp_ptr; + printf("\nflush-on-iface-down: %s\n", resp->flush_routes_on_down ? "on" : "off"); + printf("skip-events-on-iface-down: %s\n", + resp->skip_route_events_on_iface_down ? "on" : "off"); + free(resp_ptr); + } + return ret < 0 ? CMD_ERROR : CMD_SUCCESS; } @@ -210,6 +245,25 @@ static int ctx_init(struct ec_node *root) { ); if (ret < 0) return ret; + ret = CLI_COMMAND( + CONFIG_CTX(root), + "set (flush-on-iface-down FLUSH),(skip-events-on-iface-down SKIP)", + route_flush_config_set, + "Change the interface teardown behaviour.", + with_help( + "Delete the routes going out of an interface when it goes " + "administratively down.", + EC_NODE_OR("FLUSH", ec_node_str("", "on"), ec_node_str("", "off")) + ), + with_help( + "Do not emit route events for the routes deleted along with an " + "interface.", + EC_NODE_OR("SKIP", ec_node_str("", "on"), ec_node_str("", "off")) + ) + ); + if (ret < 0) + return ret; + ret = CLI_COMMAND( CONFIG_CTX(root), "[show] [(FAMILY),(vrf VRF)]", diff --git a/modules/infra/control/iface.h b/modules/infra/control/iface.h index 031255707..23a6b5978 100644 --- a/modules/infra/control/iface.h +++ b/modules/infra/control/iface.h @@ -97,6 +97,9 @@ uint16_t vrf_default_get_or_create(void); // vrf_id is the VRF interface ID. struct iface *get_vrf_iface(uint16_t vrf_id); +// Get the global interface behaviour configuration. +const struct gr_iface_config *iface_config_get(void); + // Register a reserved interface name. // If prefix is true, any name starting with the string is reserved. // If prefix is false, only the exact name is reserved. diff --git a/modules/infra/control/l3_nexthop.c b/modules/infra/control/l3_nexthop.c index 8726c7692..371d3df9a 100644 --- a/modules/infra/control/l3_nexthop.c +++ b/modules/infra/control/l3_nexthop.c @@ -143,12 +143,12 @@ static int l3_reconfig(const struct gr_nexthop_config *c) { return 0; } -void nexthop_routes_cleanup(struct nexthop *nh) { +void nexthop_routes_cleanup(struct nexthop *nh, bool notify) { const struct nexthop_af_ops *ops; for (unsigned i = 0; i < ARRAY_DIM(af_ops); i++) { ops = af_ops[i]; if (ops != NULL) - ops->cleanup_routes(nh); + ops->cleanup_routes(nh, notify); } } diff --git a/modules/infra/control/nexthop.c b/modules/infra/control/nexthop.c index 3703c2d74..9348243a5 100644 --- a/modules/infra/control/nexthop.c +++ b/modules/infra/control/nexthop.c @@ -479,7 +479,7 @@ static void nh_cleanup_interface_cb(struct nexthop *nh, void *priv) { if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS) return; // addresses are cleaned per address family } - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, !iface_config_get()->skip_route_events_on_iface_down); while (nh->ref_count) nexthop_decref(nh); } diff --git a/modules/infra/control/nexthop.h b/modules/infra/control/nexthop.h index 14fef399a..2e16f2e88 100644 --- a/modules/infra/control/nexthop.h +++ b/modules/infra/control/nexthop.h @@ -128,7 +128,7 @@ struct gr_nexthop *nexthop_to_api(const struct nexthop *, size_t *len); int nexthop_serialize(const void *obj, void **buf); // Clean all routes that reference a given nexthop. -void nexthop_routes_cleanup(struct nexthop *); +void nexthop_routes_cleanup(struct nexthop *, bool notify); // Increment the reference counter of a nexthop. void nexthop_incref(struct nexthop *); @@ -156,7 +156,7 @@ struct nexthop_af_ops { // Callback that will be invoked when a nexthop needs to be refreshed by sending a probe. int (*solicit)(struct nexthop *); // Callback that will be invoked to delete all routes which reference a given nexthop. - void (*cleanup_routes)(struct nexthop *); + void (*cleanup_routes)(struct nexthop *, bool notify); // Callback invoked by resolve() to flush held packets when the nexthop becomes // reachable. int (*resubmit)(struct rte_mbuf *, struct nexthop *); diff --git a/modules/ip/control/address.c b/modules/ip/control/address.c index c7c4322ea..a6828236e 100644 --- a/modules/ip/control/address.c +++ b/modules/ip/control/address.c @@ -174,13 +174,16 @@ int addr4_delete(uint16_t iface_id, ip4_addr_t ip, uint16_t prefixlen) { } ); - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); while (nh->ref_count > 0) nexthop_decref(nh); vec_del(addrs->nh, i); - if (vec_len(addrs->nh) == 0) + if (vec_len(addrs->nh) == 0) { vec_free(addrs->nh); + // no address left to send ARP requests from + rib4_cleanup_iface(iface_id, true); + } iface = iface_from_id(iface_id); if (iface && iface->cp_id != 0) { diff --git a/modules/ip/control/ip4.h b/modules/ip/control/ip4.h index 53daf6a97..3e8d48ce3 100644 --- a/modules/ip/control/ip4.h +++ b/modules/ip/control/ip4.h @@ -34,8 +34,16 @@ int rib4_insert( gr_nh_origin_t origin, struct nexthop *nh ); -int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t nh_type); -void rib4_cleanup(struct nexthop *); +int rib4_delete( + uint16_t vrf_id, + ip4_addr_t ip, + uint8_t prefixlen, + gr_nh_type_t nh_type, + bool notify +); +void rib4_cleanup(struct nexthop *, bool notify); +// delete the routes of an interface, except those owned by a routing daemon +void rib4_cleanup_iface(uint16_t iface_id, bool notify); typedef int (*rib4_iter_cb_t)( uint16_t vrf_id, diff --git a/modules/ip/control/route.c b/modules/ip/control/route.c index 8846d5662..c2fbb2a8a 100644 --- a/modules/ip/control/route.c +++ b/modules/ip/control/route.c @@ -284,7 +284,13 @@ int rib4_insert( return rib4_insert_or_replace(vrf_id, ip, prefixlen, origin, nh, false); } -int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t nh_type) { +int rib4_delete( + uint16_t vrf_id, + ip4_addr_t ip, + uint8_t prefixlen, + gr_nh_type_t nh_type, + bool notify +) { struct rte_fib *fib = get_fib(vrf_id); gr_nh_origin_t *o, origin; struct rte_rib_node *rn; @@ -312,7 +318,7 @@ int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t if ((ret = rte_fib_delete(fib, rte_be_to_cpu_32(ip), prefixlen)) < 0) return errno_set(-ret); - if (origin != GR_NH_ORIGIN_INTERNAL) { + if (origin != GR_NH_ORIGIN_INTERNAL && notify) { event_push( GR_EVENT_IP_ROUTE_DEL, &(const struct route4_event) { @@ -393,7 +399,7 @@ static struct api_out route4_del(const void *request, struct api_ctx *) { nh = rib4_lookup(req->vrf_id, req->dest.ip); ret = rib4_delete( - req->vrf_id, req->dest.ip, req->dest.prefixlen, nh ? nh->type : GR_NH_T_L3 + req->vrf_id, req->dest.ip, req->dest.prefixlen, nh ? nh->type : GR_NH_T_L3, true ); if ((ret == -ENOENT || ret == -ENONET) && req->missing_ok) ret = 0; @@ -537,6 +543,7 @@ struct rib4_cleanup_entry { struct rib4_cleanup_ctx { const struct nexthop *nh; + uint16_t iface_id; vec struct rib4_cleanup_entry *entries; }; @@ -544,12 +551,21 @@ static int rib4_cleanup_cb( uint16_t vrf_id, ip4_addr_t ip, uint8_t depth, - gr_nh_origin_t, + gr_nh_origin_t origin, const struct nexthop *nh, void *priv ) { struct rib4_cleanup_ctx *ctx = priv; - if (ctx->nh == NULL || nh == ctx->nh) { + bool match; + + if (ctx->iface_id != GR_IFACE_ID_UNDEF) { + // Routes installed by a routing daemon are left to their owner. + match = nh->iface_id == ctx->iface_id && origin <= GR_NH_ORIGIN_STATIC; + } else { + match = ctx->nh == NULL || nh == ctx->nh; + } + + if (match) { struct rib4_cleanup_entry entry = { .vrf_id = vrf_id, .ip = ip, @@ -561,21 +577,35 @@ static int rib4_cleanup_cb( return 0; } -void rib4_cleanup(struct nexthop *nh) { - struct rib4_cleanup_ctx ctx = { - .nh = nh, - .entries = NULL, - }; +static void rib4_cleanup_run(struct rib4_cleanup_ctx *ctx, bool notify) { struct rib4_iterator iter = { .max_count = 0, .skip_internal = false, .cb = rib4_cleanup_cb, - .priv = &ctx, + .priv = ctx, }; rib4_iter(GR_VRF_ID_UNDEF, &iter); - vec_foreach_ref (struct rib4_cleanup_entry *r, ctx.entries) - rib4_delete(r->vrf_id, r->ip, r->depth, r->type); - vec_free(ctx.entries); + vec_foreach_ref (struct rib4_cleanup_entry *r, ctx->entries) + rib4_delete(r->vrf_id, r->ip, r->depth, r->type, notify); + vec_free(ctx->entries); +} + +void rib4_cleanup(struct nexthop *nh, bool notify) { + struct rib4_cleanup_ctx ctx = { + .nh = nh, + .iface_id = GR_IFACE_ID_UNDEF, + .entries = NULL, + }; + rib4_cleanup_run(&ctx, notify); +} + +void rib4_cleanup_iface(uint16_t iface_id, bool notify) { + struct rib4_cleanup_ctx ctx = { + .nh = NULL, + .iface_id = iface_id, + .entries = NULL, + }; + rib4_cleanup_run(&ctx, notify); } METRIC_GAUGE(m_routes, "rib4_routes", "Number of IPv4 routes by origin."); @@ -848,7 +878,7 @@ static void fib4_fini(struct iface *vrf) { rib4_iter_vrf(rte_fib_get_rib(fib), vrf->id, &iter); vec_foreach_ref (struct rib4_cleanup_entry *r, ctx.entries) - rib4_delete(r->vrf_id, r->ip, r->depth, r->type); + rib4_delete(r->vrf_id, r->ip, r->depth, r->type, true); vec_free(ctx.entries); iface_info_vrf(vrf)->fib4 = NULL; @@ -862,7 +892,7 @@ static struct api_out fib4_default_set(const void *request, struct api_ctx *) { const struct gr_ip4_fib_default_set_req *req = request; if (req->max_routes == 0) - return api_out(EINVAL, 0, NULL); + return api_out(0, 0, NULL); if (req->max_routes != max_routes_default) { LOG(INFO, "IPv4 default max_routes %u -> %u", max_routes_default, req->max_routes); @@ -878,6 +908,20 @@ static const struct vrf_fib_ops fib4_ops = { .fini = fib4_fini, }; +static void iface_down_cb(uint32_t /*ev_type*/, const void *obj) { + const struct iface *iface = obj; + + // carrier loss alone is not a trigger + if (iface->flags & GR_IFACE_F_UP) + return; + const struct gr_iface_config *conf = iface_config_get(); + + if (!conf->flush_routes_on_down) + return; + + rib4_cleanup_iface(iface->id, !conf->skip_route_events_on_iface_down); +} + RTE_INIT(control_ip_init) { api_handler(GR_IP4_ROUTE_ADD, route4_add); api_handler(GR_IP4_ROUTE_DEL, route4_del); @@ -890,4 +934,6 @@ RTE_INIT(control_ip_init) { module_register(&route4_module); metrics_register(&rib4_collector); vrf_fib_ops_register(GR_AF_IP4, &fib4_ops); + event_subscribe(GR_EVENT_IFACE_POST_RECONFIG, iface_down_cb); + event_subscribe(GR_EVENT_IFACE_STATUS_DOWN, iface_down_cb); } diff --git a/modules/ip6/control/address.c b/modules/ip6/control/address.c index a2febf772..7c23cc1b5 100644 --- a/modules/ip6/control/address.c +++ b/modules/ip6/control/address.c @@ -307,7 +307,7 @@ int addr6_delete(uint16_t iface_id, const struct rte_ipv6_addr *ip, uint8_t pref } ); - nexthop_routes_cleanup(nh); + nexthop_routes_cleanup(nh, true); while (nh->ref_count > 0) nexthop_decref(nh); diff --git a/modules/ip6/control/ip6.h b/modules/ip6/control/ip6.h index 83fb9c975..20b4b33af 100644 --- a/modules/ip6/control/ip6.h +++ b/modules/ip6/control/ip6.h @@ -68,9 +68,12 @@ int rib6_delete( uint16_t iface_id, const struct rte_ipv6_addr *, uint8_t prefixlen, - gr_nh_type_t nh_type + gr_nh_type_t nh_type, + bool notify ); -void rib6_cleanup(struct nexthop *); +void rib6_cleanup(struct nexthop *, bool notify); +// delete the routes of an interface, except those owned by a routing daemon +void rib6_cleanup_iface(uint16_t iface_id, bool notify); struct nexthop *rib6_lookup(uint16_t vrf_id, uint16_t iface_id, const struct rte_ipv6_addr *); struct nexthop *rib6_lookup_exact( uint16_t vrf_id, diff --git a/modules/ip6/control/route.c b/modules/ip6/control/route.c index a775b8ffe..0260df4ab 100644 --- a/modules/ip6/control/route.c +++ b/modules/ip6/control/route.c @@ -312,7 +312,8 @@ int rib6_delete( uint16_t iface_id, const struct rte_ipv6_addr *ip, uint8_t prefixlen, - gr_nh_type_t nh_type + gr_nh_type_t nh_type, + bool notify ) { struct rte_fib6 *fib = get_fib6(vrf_id); const struct rte_ipv6_addr *scoped_ip; @@ -344,7 +345,7 @@ int rib6_delete( if ((ret = rte_fib6_delete(fib, scoped_ip, prefixlen)) < 0) return errno_set(-ret); - if (origin != GR_NH_ORIGIN_INTERNAL) { + if (origin != GR_NH_ORIGIN_INTERNAL && notify) { event_push( GR_EVENT_IP6_ROUTE_DEL, &(const struct route6_event) { @@ -438,7 +439,8 @@ static struct api_out route6_del(const void *request, struct api_ctx *) { GR_IFACE_ID_UNDEF, &req->dest.ip, req->dest.prefixlen, - nh ? nh->type : GR_NH_T_L3 + nh ? nh->type : GR_NH_T_L3, + true ); if ((ret == -ENOENT || ret == -ENONET) && req->missing_ok) ret = 0; @@ -579,6 +581,7 @@ struct rib6_cleanup_entry { struct rib6_cleanup_ctx { const struct nexthop *nh; + uint16_t iface_id; vec struct rib6_cleanup_entry *entries; }; @@ -586,12 +589,21 @@ static int rib6_cleanup_cb( uint16_t vrf_id, const struct rte_ipv6_addr *ip, uint8_t depth, - gr_nh_origin_t, + gr_nh_origin_t origin, const struct nexthop *nh, void *priv ) { struct rib6_cleanup_ctx *ctx = priv; - if (ctx->nh == NULL || nh == ctx->nh) { + bool match; + + if (ctx->iface_id != GR_IFACE_ID_UNDEF) { + // Routes installed by a routing daemon are left to their owner. + match = nh->iface_id == ctx->iface_id && origin <= GR_NH_ORIGIN_STATIC; + } else { + match = ctx->nh == NULL || nh == ctx->nh; + } + + if (match) { struct rib6_cleanup_entry entry = { .vrf_id = vrf_id, .iface_id = nh->iface_id, @@ -604,21 +616,35 @@ static int rib6_cleanup_cb( return 0; } -void rib6_cleanup(struct nexthop *nh) { - struct rib6_cleanup_ctx ctx = { - .nh = nh, - .entries = NULL, - }; +static void rib6_cleanup_run(struct rib6_cleanup_ctx *ctx, bool notify) { struct rib6_iterator iter = { .max_count = 0, .skip_internal = false, .cb = rib6_cleanup_cb, - .priv = &ctx, + .priv = ctx, }; rib6_iter(GR_VRF_ID_UNDEF, &iter); - vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx.entries) - rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type); - vec_free(ctx.entries); + vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx->entries) + rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type, notify); + vec_free(ctx->entries); +} + +void rib6_cleanup(struct nexthop *nh, bool notify) { + struct rib6_cleanup_ctx ctx = { + .nh = nh, + .iface_id = GR_IFACE_ID_UNDEF, + .entries = NULL, + }; + rib6_cleanup_run(&ctx, notify); +} + +void rib6_cleanup_iface(uint16_t iface_id, bool notify) { + struct rib6_cleanup_ctx ctx = { + .nh = NULL, + .iface_id = iface_id, + .entries = NULL, + }; + rib6_cleanup_run(&ctx, notify); } METRIC_GAUGE(m_routes, "rib6_routes", "Number of IPv6 routes by origin."); @@ -884,7 +910,7 @@ static void fib6_fini(struct iface *vrf) { rib6_iter_vrf(rte_fib6_get_rib(fib), vrf->id, &iter); vec_foreach_ref (const struct rib6_cleanup_entry *r, ctx.entries) - rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type); + rib6_delete(r->vrf_id, r->iface_id, &r->ip, r->depth, r->type, true); vec_free(ctx.entries); iface_info_vrf(vrf)->fib6 = NULL; @@ -899,7 +925,7 @@ static struct api_out fib6_default_set(const void *request, struct api_ctx *) { const struct gr_ip6_fib_default_set_req *req = request; if (req->max_routes == 0) - return api_out(EINVAL, 0, NULL); + return api_out(0, 0, NULL); if (req->max_routes != max_routes_default) { LOG(INFO, "IPv6 default max_routes %u -> %u", max_routes_default, req->max_routes); @@ -915,6 +941,20 @@ static const struct vrf_fib_ops fib6_ops = { .fini = fib6_fini, }; +static void iface_down_cb(uint32_t /*ev_type*/, const void *obj) { + const struct iface *iface = obj; + + // carrier loss alone is not a trigger + if (iface->flags & GR_IFACE_F_UP) + return; + const struct gr_iface_config *conf = iface_config_get(); + + if (!conf->flush_routes_on_down) + return; + + rib6_cleanup_iface(iface->id, !conf->skip_route_events_on_iface_down); +} + RTE_INIT(control_ip_init) { api_handler(GR_IP6_ROUTE_ADD, route6_add); api_handler(GR_IP6_ROUTE_DEL, route6_del); @@ -927,4 +967,6 @@ RTE_INIT(control_ip_init) { module_register(&route6_module); metrics_register(&rib6_collector); vrf_fib_ops_register(GR_AF_IP6, &fib6_ops); + event_subscribe(GR_EVENT_IFACE_POST_RECONFIG, iface_down_cb); + event_subscribe(GR_EVENT_IFACE_STATUS_DOWN, iface_down_cb); } diff --git a/modules/policy/api/dnat44.c b/modules/policy/api/dnat44.c index 2c8f00903..ee56f0d1c 100644 --- a/modules/policy/api/dnat44.c +++ b/modules/policy/api/dnat44.c @@ -176,7 +176,7 @@ static struct api_out dnat44_del(const void *request, struct api_ctx *) { if (iface == NULL) return api_out(ENODEV, 0, NULL); - ret = rib4_delete(iface->vrf_id, req->match, 32, GR_NH_T_DNAT); + ret = rib4_delete(iface->vrf_id, req->match, 32, GR_NH_T_DNAT, true); if (ret == -ENOENT && req->missing_ok) ret = 0; diff --git a/smoke/addr_del_kernel_route_frr_test.sh b/smoke/addr_del_kernel_route_frr_test.sh new file mode 100755 index 000000000..773924089 --- /dev/null +++ b/smoke/addr_del_kernel_route_frr_test.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Maxime Leroy, Free Mobile + +# grout and zebra must agree on the routes of an interface whose addresses +# are deleted. +# +# The addresses and routes are configured with grcli on purpose: an address +# configured in FRR keeps its connected entry flagged ZEBRA_IFC_CONFIGURED, +# and zebra would not consider the interface address-less. + +. $(dirname $0)/_init_frr.sh + +prefix4=203.0.113.0/24 +prefix6=2001:db8:beef::/64 +gw4=192.168.100.2 +gw6=fd00:ba4::2 + +# assert_agree +# +# Check that grout and zebra hold the same view of , and that this +# view is the expected one. Retries to let the notification propagate. +assert_agree() { + local prefix="$1" + local expected="$2" + local ctx="$3" + local show="show ip route" + local in_grout in_zebra tries=20 + + [ "${prefix#*:}" != "$prefix" ] && show="show ipv6 route" + + while [ "$tries" -gt 0 ]; do + in_grout=no + in_zebra=no + if grcli -j route show | jq -e \ + ".[] | select(.destination == \"$prefix\")" >/dev/null 2>&1; then + in_grout=yes + fi + if vtysh -c "$show $prefix" 2>/dev/null | grep -qE 'Known via "kernel"'; then + in_zebra=yes + fi + [ "$in_grout" = "$expected" ] && [ "$in_zebra" = "$expected" ] && break + tries=$((tries - 1)) + sleep 0.2 + done + + [ "$in_grout" = "$in_zebra" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, tables disagree" + [ "$in_grout" = "$expected" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, expected $expected" +} + +create_interface p0 + +netns_add n0 +move_to_netns x-p0 n0 +ip -n n0 addr add $gw4/24 dev x-p0 +ip -n n0 addr add $gw6/64 dev x-p0 + +# Addresses and routes owned by grout: zebra learns them through the +# plugin, so the connected entries are not flagged ZEBRA_IFC_CONFIGURED +# and really leave ifp->connected when deleted. +grcli address add 192.168.0.1/24 iface p0 +grcli address add 192.168.100.1/24 iface p0 +grcli address add fd00:f00::1/64 iface p0 +grcli address add fd00:ba4::1/64 iface p0 +grcli route add $prefix4 via $gw4 +grcli route add $prefix6 via $gw6 + +assert_agree $prefix4 yes "after route add" +assert_agree $prefix6 yes "after route add" + +# Deleting an address which is not the last one of its family changes +# nothing. +grcli address del 192.168.0.1/24 iface p0 +grcli address del fd00:f00::1/64 iface p0 + +assert_agree $prefix4 yes "after deleting a non-last address" +assert_agree $prefix6 yes "after deleting a non-last address" + +# Deleting the last IPv6 address keeps both routes: the interface still +# has its link-local address to source neighbor solicitations from. +grcli address del fd00:ba4::1/64 iface p0 + +assert_agree $prefix6 yes "after deleting the last IPv6 address" +assert_agree $prefix4 yes "after deleting the last IPv6 address" + +# Deleting the last IPv4 address flushes the IPv4 routes of the +# interface, and only those. +grcli address del 192.168.100.1/24 iface p0 + +assert_agree $prefix4 no "after deleting the last IPv4 address" +assert_agree $prefix6 yes "after deleting the last IPv4 address" + +true diff --git a/smoke/iface_down_kernel_route_frr_test.sh b/smoke/iface_down_kernel_route_frr_test.sh new file mode 100755 index 000000000..30fbc8e41 --- /dev/null +++ b/smoke/iface_down_kernel_route_frr_test.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2026 Maxime Leroy, Free Mobile + +# grout and zebra must agree on the routes of an interface which goes +# administratively down, and both must leave them alone on carrier loss. +# +# The addresses and routes are configured with grcli on purpose, see +# addr_del_kernel_route_frr_test.sh. + +. $(dirname $0)/_init_frr.sh + +prefix4=203.0.113.0/24 +prefix6=2001:db8:beef::/64 +gw4=192.168.0.2 +gw6=fd00:ba4::2 + +# assert_agree +assert_agree() { + local prefix="$1" + local expected="$2" + local ctx="$3" + local show="show ip route" + local in_grout in_zebra tries=20 + + [ "${prefix#*:}" != "$prefix" ] && show="show ipv6 route" + + while [ "$tries" -gt 0 ]; do + in_grout=no + in_zebra=no + if grcli -j route show | jq -e \ + ".[] | select(.destination == \"$prefix\")" >/dev/null 2>&1; then + in_grout=yes + fi + if vtysh -c "$show $prefix" 2>/dev/null | grep -qE 'Known via "kernel"'; then + in_zebra=yes + fi + [ "$in_grout" = "$expected" ] && [ "$in_zebra" = "$expected" ] && break + tries=$((tries - 1)) + sleep 0.2 + done + + [ "$in_grout" = "$in_zebra" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, tables disagree" + [ "$in_grout" = "$expected" ] || + fail "$prefix $ctx: grout=$in_grout zebra=$in_zebra, expected $expected" +} + +create_interface p0 + +netns_add n0 +move_to_netns x-p0 n0 +ip -n n0 addr add $gw4/24 dev x-p0 +ip -n n0 addr add $gw6/64 dev x-p0 + +grcli address add 192.168.0.1/24 iface p0 +grcli address add fd00:ba4::1/64 iface p0 +grcli route add $prefix4 via $gw4 +grcli route add $prefix6 via $gw6 + +# The plugin enables the flush when it connects to grout. +grcli route config show | grep -q "flush-on-iface-down: on" || + fail "the plugin did not enable flush-on-iface-down" + +assert_agree $prefix4 yes "after route add" +assert_agree $prefix6 yes "after route add" + +# Carrier loss only: the interface stays up, nothing is flushed. +ip -n n0 link set x-p0 down + +assert_agree $prefix4 yes "after carrier loss" +assert_agree $prefix6 yes "after carrier loss" + +ip -n n0 link set x-p0 up + +# Administratively down: both families are flushed on both sides. +grcli interface set port p0 down + +assert_agree $prefix4 no "after setting p0 down" +assert_agree $prefix6 no "after setting p0 down" + +true