ip,ip6,frr: flush routes like zebra on address and iface teardown

frr/zebra_dplane_grout.c

@@ -710,6 +710,16 @@ static void dplane_grout_connect(struct event *) {
 		&grout_ctx.dg_t_dplane_update
 	);
 
+	// redone on every connection: a restarted grout comes back with its defaults
+	struct gr_iface_config_set_req req = {
+		.flush_routes_on_down = true,
+		.skip_route_events_on_iface_down = true,
+		.set_attrs = GR_IFACE_CONFIG_SET_FLUSH_ROUTES | GR_IFACE_CONFIG_SET_SKIP_EVENTS,
+	};
+	if (gr_api_client_send_recv(grout_ctx.client, GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL)
+	    < 0)
+		gr_log_err("GR_IFACE_CONFIG_SET: %s", strerror(errno));
+
 	gr_log_notice("connected, monitoring iface/ip events");
 }
 

modules/dhcp/control/client.c

@@ -198,7 +198,7 @@ static void dhcp_expire_callback(evutil_socket_t, short, void *arg) {
 	if (client->offered_ip != 0 && client->prefixlen != 0)
 		addr4_delete(iface->id, client->offered_ip, client->prefixlen);
 	if (client->router_ip != 0)
-		rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3);
+		rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true);
 
 	client->state = DHCP_STATE_INIT;
 	client->offered_ip = 0;
@@ -370,7 +370,7 @@ void dhcp_input_cb(void *obj, uintptr_t, const struct control_queue_drain *drain
 			if (client->offered_ip != 0 && client->prefixlen != 0)
 				addr4_delete(iface->id, client->offered_ip, client->prefixlen);
 			if (client->router_ip != 0)
-				rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3);
+				rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true);
 
 			client->state = DHCP_STATE_INIT;
 			client->offered_ip = 0;
@@ -454,7 +454,7 @@ static int dhcp_stop(uint16_t iface_id) {
 	if (client->offered_ip != 0 && client->prefixlen != 0)
 		addr4_delete(iface->id, client->offered_ip, client->prefixlen);
 	if (client->router_ip != 0)
-		rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3);
+		rib4_delete(iface->vrf_id, 0, 0, GR_NH_T_L3, true);
 
 	dhcp_cancel_timers(client);
 

modules/infra/api/gr_infra.h

@@ -218,6 +218,14 @@ struct gr_port_rxq_map {
 	uint16_t enabled;
 };
 
+// Global interface behaviour configuration.
+struct gr_iface_config {
+	// Delete the routes of an interface when it goes administratively down.
+	bool flush_routes_on_down;
+	// Do not emit route events for the routes deleted with an interface.
+	bool skip_route_events_on_iface_down;
+};
+
 // Infrastructure statistics entry.
 struct gr_stat {
 	char name[64];
@@ -252,6 +260,8 @@ enum gr_infra_requests : uint32_t {
 	GR_IFACE_MAC_DEL,
 	GR_IFACE_MAC_LIST,
 	GR_IFACE_MAC_SET,
+	GR_IFACE_CONFIG_GET,
+	GR_IFACE_CONFIG_SET,
 };
 
 enum gr_infra_events : uint32_t {
@@ -363,6 +373,23 @@ struct gr_iface_set_req {
 
 GR_REQ(GR_IFACE_SET, struct gr_iface_set_req, struct gr_empty);
 
+// Interface configuration attributes flags.
+#define GR_IFACE_CONFIG_SET_FLUSH_ROUTES GR_BIT64(0)
+#define GR_IFACE_CONFIG_SET_SKIP_EVENTS GR_BIT64(1)
+
+struct gr_iface_config_set_req {
+	BASE(gr_iface_config);
+	uint64_t set_attrs; // Bit mask of GR_IFACE_CONFIG_SET_*.
+};
+
+GR_REQ(GR_IFACE_CONFIG_SET, struct gr_iface_config_set_req, struct gr_empty);
+
+struct gr_iface_config_get_resp {
+	BASE(gr_iface_config);
+};
+
+GR_REQ(GR_IFACE_CONFIG_GET, struct gr_empty, struct gr_iface_config_get_resp);
+
 // Get interface statistics.
 struct gr_iface_stats {
 	uint16_t iface_id;

modules/infra/api/iface.c

@@ -323,6 +323,31 @@ static void iface_metrics_collect(struct metrics_writer *w) {
 	}
 }
 
+static struct gr_iface_config iface_config;
+
+const struct gr_iface_config *iface_config_get(void) {
+	return &iface_config;
+}
+
+static struct api_out config_get(const void *, struct api_ctx *) {
+	struct gr_iface_config_get_resp *resp = calloc(1, sizeof(*resp));
+	if (resp == NULL)
+		return api_out(ENOMEM, 0, NULL);
+	resp->base = iface_config;
+	return api_out(0, sizeof(*resp), resp);
+}
+
+static struct api_out config_set(const void *request, struct api_ctx *) {
+	const struct gr_iface_config_set_req *req = request;
+
+	if (req->set_attrs & GR_IFACE_CONFIG_SET_FLUSH_ROUTES)
+		iface_config.flush_routes_on_down = req->flush_routes_on_down;
+	if (req->set_attrs & GR_IFACE_CONFIG_SET_SKIP_EVENTS)
+		iface_config.skip_route_events_on_iface_down = req->skip_route_events_on_iface_down;
+
+	return api_out(0, 0, NULL);
+}
+
 static struct metrics_collector iface_collector = {
 	.name = "iface",
 	.collect = iface_metrics_collect,
@@ -338,6 +363,8 @@ RTE_INIT(infra_api_init) {
 	api_handler(GR_IFACE_MAC_LIST, iface_mac_list);
 	api_handler(GR_IFACE_MAC_SET, iface_mac_set);
 	api_handler(GR_IFACE_SET, iface_set);
+	api_handler(GR_IFACE_CONFIG_GET, config_get);
+	api_handler(GR_IFACE_CONFIG_SET, config_set);
 	event_serializer(GR_EVENT_IFACE_ADD, iface_event_serialize);
 	event_serializer(GR_EVENT_IFACE_POST_ADD, iface_event_serialize);
 	event_serializer(GR_EVENT_IFACE_PRE_REMOVE, iface_event_serialize);

modules/infra/api/nexthop.c

@@ -67,7 +67,7 @@ static struct api_out nh_del(const void *request, struct api_ctx *) {
 			return api_out(EBUSY, 0, NULL);
 	}
 
-	nexthop_routes_cleanup(nh);
+	nexthop_routes_cleanup(nh, true);
 	// The nexthop *may* still have one ref_count when it has been created
 	// manually from the API (see nh_add()). Implicit nexthops created when
 	// creating a gateway route will not have that extra ref_count.
@@ -142,7 +142,7 @@ static struct api_out nh_flush(const void *request, struct api_ctx *) {
 			if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS)
 				continue;
 		}
-		nexthop_routes_cleanup(nh);
+		nexthop_routes_cleanup(nh, true);
 		while (nh->ref_count > 0)
 			nexthop_decref(nh);
 	}

modules/infra/cli/route.c

@@ -97,6 +97,28 @@ static cmd_status_t route_list(struct gr_api_client *c, const struct ec_pnode *p
 	return ret < 0 ? CMD_ERROR : CMD_SUCCESS;
 }
 
+static cmd_status_t route_flush_config_set(struct gr_api_client *c, const struct ec_pnode *p) {
+	struct gr_iface_config_set_req req = {0};
+	const char *flush, *skip;
+
+	flush = arg_str(p, "FLUSH");
+	if (flush != NULL) {
+		req.flush_routes_on_down = strcmp(flush, "on") == 0;
+		req.set_attrs |= GR_IFACE_CONFIG_SET_FLUSH_ROUTES;
+	}
+
+	skip = arg_str(p, "SKIP");
+	if (skip != NULL) {
+		req.skip_route_events_on_iface_down = strcmp(skip, "on") == 0;
+		req.set_attrs |= GR_IFACE_CONFIG_SET_SKIP_EVENTS;
+	}
+
+	if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_SET, sizeof(req), &req, NULL) < 0)
+		return CMD_ERROR;
+
+	return CMD_SUCCESS;
+}
+
 static cmd_status_t route_config_set(struct gr_api_client *c, const struct ec_pnode *p) {
 	struct cli_route_ops *ops;
 	cmd_status_t ret;
@@ -140,6 +162,19 @@ static cmd_status_t route_config_show(struct gr_api_client *c, const struct ec_p
 
 	gr_table_free(table);
 
+	if (ret == 0) {
+		const struct gr_iface_config_get_resp *resp;
+		void *resp_ptr = NULL;
+
+		if (gr_api_client_send_recv(c, GR_IFACE_CONFIG_GET, 0, NULL, &resp_ptr) < 0)
+			return CMD_ERROR;
+		resp = resp_ptr;
+		printf("\nflush-on-iface-down: %s\n", resp->flush_routes_on_down ? "on" : "off");
+		printf("skip-events-on-iface-down: %s\n",
+		       resp->skip_route_events_on_iface_down ? "on" : "off");
+		free(resp_ptr);
+	}
+
 	return ret < 0 ? CMD_ERROR : CMD_SUCCESS;
 }
 
@@ -210,6 +245,25 @@ static int ctx_init(struct ec_node *root) {
 	);
 	if (ret < 0)
 		return ret;
+	ret = CLI_COMMAND(
+		CONFIG_CTX(root),
+		"set (flush-on-iface-down FLUSH),(skip-events-on-iface-down SKIP)",
+		route_flush_config_set,
+		"Change the interface teardown behaviour.",
+		with_help(
+			"Delete the routes going out of an interface when it goes "
+			"administratively down.",
+			EC_NODE_OR("FLUSH", ec_node_str("", "on"), ec_node_str("", "off"))
+		),
+		with_help(
+			"Do not emit route events for the routes deleted along with an "
+			"interface.",
+			EC_NODE_OR("SKIP", ec_node_str("", "on"), ec_node_str("", "off"))
+		)
+	);
+	if (ret < 0)
+		return ret;
+
 	ret = CLI_COMMAND(
 		CONFIG_CTX(root),
 		"[show] [(FAMILY),(vrf VRF)]",

modules/infra/control/iface.h

@@ -97,6 +97,9 @@ uint16_t vrf_default_get_or_create(void);
 // vrf_id is the VRF interface ID.
 struct iface *get_vrf_iface(uint16_t vrf_id);
 
+// Get the global interface behaviour configuration.
+const struct gr_iface_config *iface_config_get(void);
+
 // Register a reserved interface name.
 // If prefix is true, any name starting with the string is reserved.
 // If prefix is false, only the exact name is reserved.

modules/infra/control/l3_nexthop.c

@@ -143,12 +143,12 @@ static int l3_reconfig(const struct gr_nexthop_config *c) {
 	return 0;
 }
 
-void nexthop_routes_cleanup(struct nexthop *nh) {
+void nexthop_routes_cleanup(struct nexthop *nh, bool notify) {
 	const struct nexthop_af_ops *ops;
 	for (unsigned i = 0; i < ARRAY_DIM(af_ops); i++) {
 		ops = af_ops[i];
 		if (ops != NULL)
-			ops->cleanup_routes(nh);
+			ops->cleanup_routes(nh, notify);
 	}
 }
 

modules/infra/control/nexthop.c

@@ -479,7 +479,7 @@ static void nh_cleanup_interface_cb(struct nexthop *nh, void *priv) {
 			if ((l3->flags & NH_LOCAL_ADDR_FLAGS) == NH_LOCAL_ADDR_FLAGS)
 				return; // addresses are cleaned per address family
 		}
-		nexthop_routes_cleanup(nh);
+		nexthop_routes_cleanup(nh, !iface_config_get()->skip_route_events_on_iface_down);
 		while (nh->ref_count)
 			nexthop_decref(nh);
 	}

modules/infra/control/nexthop.h

@@ -128,7 +128,7 @@ struct gr_nexthop *nexthop_to_api(const struct nexthop *, size_t *len);
 int nexthop_serialize(const void *obj, void **buf);
 
 // Clean all routes that reference a given nexthop.
-void nexthop_routes_cleanup(struct nexthop *);
+void nexthop_routes_cleanup(struct nexthop *, bool notify);
 
 // Increment the reference counter of a nexthop.
 void nexthop_incref(struct nexthop *);
@@ -156,7 +156,7 @@ struct nexthop_af_ops {
 	// Callback that will be invoked when a nexthop needs to be refreshed by sending a probe.
 	int (*solicit)(struct nexthop *);
 	// Callback that will be invoked to delete all routes which reference a given nexthop.
-	void (*cleanup_routes)(struct nexthop *);
+	void (*cleanup_routes)(struct nexthop *, bool notify);
 	// Callback invoked by resolve() to flush held packets when the nexthop becomes
 	// reachable.
 	int (*resubmit)(struct rte_mbuf *, struct nexthop *);

modules/ip/control/address.c

@@ -174,13 +174,16 @@ int addr4_delete(uint16_t iface_id, ip4_addr_t ip, uint16_t prefixlen) {
 		}
 	);
 
-	nexthop_routes_cleanup(nh);
+	nexthop_routes_cleanup(nh, true);
 	while (nh->ref_count > 0)
 		nexthop_decref(nh);
 
 	vec_del(addrs->nh, i);
-	if (vec_len(addrs->nh) == 0)
+	if (vec_len(addrs->nh) == 0) {
 		vec_free(addrs->nh);
+		// no address left to send ARP requests from
+		rib4_cleanup_iface(iface_id, true);
+	}
 
 	iface = iface_from_id(iface_id);
 	if (iface && iface->cp_id != 0) {

modules/ip/control/ip4.h

@@ -34,8 +34,16 @@ int rib4_insert(
 	gr_nh_origin_t origin,
 	struct nexthop *nh
 );
-int rib4_delete(uint16_t vrf_id, ip4_addr_t ip, uint8_t prefixlen, gr_nh_type_t nh_type);
-void rib4_cleanup(struct nexthop *);
+int rib4_delete(
+	uint16_t vrf_id,
+	ip4_addr_t ip,
+	uint8_t prefixlen,
+	gr_nh_type_t nh_type,
+	bool notify
+);
+void rib4_cleanup(struct nexthop *, bool notify);
+// delete the routes of an interface, except those owned by a routing daemon
+void rib4_cleanup_iface(uint16_t iface_id, bool notify);
 
 typedef int (*rib4_iter_cb_t)(
 	uint16_t vrf_id,