diff --git a/0230-optimize-do_close.patch b/0230-optimize-do_close.patch new file mode 100644 index 0000000..4267927 --- /dev/null +++ b/0230-optimize-do_close.patch @@ -0,0 +1,36 @@ +From 22405fd936105e4178a617d46077c871835ac6d5 Mon Sep 17 00:00:00 2001 +From: wangweizZZ +Date: Mon, 27 Mar 2023 21:46:59 +0800 +Subject: [PATCH] optimize do_close + +--- + src/lstack/api/lstack_wrap.c | 10 ++++------ + 1 file changed, 4 insertions(+), 6 deletions(-) + +diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c +index aacb30c..ae08bfe 100644 +--- a/src/lstack/api/lstack_wrap.c ++++ b/src/lstack/api/lstack_wrap.c +@@ -521,15 +521,13 @@ static inline ssize_t do_sendmsg(int32_t s, const struct msghdr *message, int32_ + + static inline int32_t do_close(int32_t s) + { +- struct lwip_sock *sock = get_socket_by_fd(s); +- if (likely(posix_api->ues_posix == 0) && sock && sock->wakeup && sock->wakeup->epollfd == s) { +- return lstack_epoll_close(s); +- } +- ++ struct lwip_sock *sock = NULL; + if (select_path(s, &sock) == PATH_KERNEL) { + return posix_api->close_fn(s); + } +- ++ if (sock && sock->wakeup && sock->wakeup->epollfd == s) { ++ return lstack_epoll_close(s); ++ } + return stack_broadcast_close(s); + } + +-- +2.23.0 + diff --git a/0231-fix-config-flow-rule-race.patch b/0231-fix-config-flow-rule-race.patch new file mode 100644 index 0000000..3f29637 --- /dev/null +++ b/0231-fix-config-flow-rule-race.patch @@ -0,0 +1,121 @@ +From b393509c3e7352ddc2457bca766a7dcff579b3f0 Mon Sep 17 00:00:00 2001 +From: jiangheng12 +Date: Thu, 30 Mar 2023 16:20:34 +0800 +Subject: [PATCH] fix config flow rule race + +--- + src/common/gazelle_dfx_msg.h | 2 +- + src/lstack/netif/lstack_ethdev.c | 6 +++--- + src/lstack/netif/lstack_vdev.c | 21 ++++++++++----------- + 3 files changed, 14 insertions(+), 15 deletions(-) + +diff --git a/src/common/gazelle_dfx_msg.h b/src/common/gazelle_dfx_msg.h +index 9105871..e4da687 100644 +--- a/src/common/gazelle_dfx_msg.h ++++ b/src/common/gazelle_dfx_msg.h +@@ -200,7 +200,7 @@ struct gazelle_stat_low_power_info { + }; + + #define RTE_ETH_XSTATS_NAME_SIZE 64 +-#define RTE_ETH_XSTATS_MAX_LEN 128 ++#define RTE_ETH_XSTATS_MAX_LEN 256 + struct nic_eth_xstats_name { + char name[RTE_ETH_XSTATS_NAME_SIZE]; + }; +diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c +index 8d05bdd..52aa9a8 100644 +--- a/src/lstack/netif/lstack_ethdev.c ++++ b/src/lstack/netif/lstack_ethdev.c +@@ -152,7 +152,7 @@ int32_t eth_dev_poll(void) + } + + /* flow rule map */ +-#define RULE_KEY_LEN 22 ++#define RULE_KEY_LEN 23 + struct flow_rule { + char rule_key[RULE_KEY_LEN]; + struct rte_flow *flow; +@@ -322,7 +322,7 @@ create_flow_director(uint16_t port_id, uint16_t queue_id, uint32_t src_ip, uint3 + void config_flow_director(uint16_t queue_id, uint32_t src_ip, uint32_t dst_ip, uint16_t src_port, uint16_t dst_port){ + + uint16_t port_id = get_port_id(); +- char rule_key[RULE_KEY_LEN]; ++ char rule_key[RULE_KEY_LEN] = {0}; + sprintf(rule_key,"%u_%u_%u",src_ip,src_port,dst_port); + struct flow_rule *fl_exist = find_rule(rule_key); + if(fl_exist != NULL){ +@@ -346,7 +346,7 @@ void config_flow_director(uint16_t queue_id, uint32_t src_ip, uint32_t dst_ip, u + void delete_flow_director(uint32_t dst_ip, uint16_t src_port, uint16_t dst_port) + { + uint16_t port_id = get_port_id(); +- char rule_key[RULE_KEY_LEN]; ++ char rule_key[RULE_KEY_LEN] = {0}; + sprintf(rule_key,"%u_%u_%u",dst_ip,dst_port,src_port); + struct flow_rule *fl = find_rule(rule_key); + +diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c +index ba0db39..aef6035 100644 +--- a/src/lstack/netif/lstack_vdev.c ++++ b/src/lstack/netif/lstack_vdev.c +@@ -152,10 +152,10 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) + } + + if (!use_ltran() && get_global_cfg_params()->tuple_filter) { +- if(type == REG_RING_TCP_LISTEN_CLOSE){ ++ if (type == REG_RING_TCP_LISTEN_CLOSE) { + if (get_global_cfg_params()->is_primary) { + delete_user_process_port(qtuple->src_port, PORT_LISTEN); +- }else{ ++ } else { + transfer_add_or_delete_listen_port_to_process0(qtuple->src_port,get_global_cfg_params()->process_idx, 0); + } + } +@@ -165,10 +165,10 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) + delete_user_process_port(qtuple->src_port, PORT_CONNECT); + uint16_t queue_id = get_protocol_stack()->queue_id; + if (queue_id != 0) { +- delete_flow_director(qtuple->dst_ip, qtuple->src_port, qtuple->dst_port); ++ transfer_delete_rule_info_to_process0(qtuple->dst_ip, qtuple->src_port, qtuple->dst_port); + } +- }else{ +- transfer_delete_rule_info_to_process0(qtuple->dst_ip,qtuple->src_port,qtuple->dst_port); ++ } else { ++ transfer_delete_rule_info_to_process0(qtuple->dst_ip, qtuple->src_port, qtuple->dst_port); + } + } + +@@ -177,18 +177,18 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) + if (get_global_cfg_params()->is_primary){ + add_user_process_port(qtuple->src_port, get_global_cfg_params()->process_idx, PORT_CONNECT); + if (queue_id != 0) { +- config_flow_director(queue_id, qtuple->dst_ip, qtuple->src_ip, qtuple->dst_port, qtuple->src_port); ++ transfer_create_rule_info_to_process0(queue_id, qtuple->src_ip, qtuple->dst_ip, qtuple->src_port, qtuple->dst_port); + } +- }else { ++ } else { + transfer_create_rule_info_to_process0(queue_id, qtuple->src_ip, qtuple->dst_ip, qtuple->src_port, qtuple->dst_port); + } + } + +- if (type == REG_RING_TCP_LISTEN){ ++ if (type == REG_RING_TCP_LISTEN) { + if (get_global_cfg_params()->is_primary) { + add_user_process_port(qtuple->src_port, get_global_cfg_params()->process_idx, PORT_LISTEN); +- }else { +- transfer_add_or_delete_listen_port_to_process0(qtuple->src_port,get_global_cfg_params()->process_idx, 1); ++ } else { ++ transfer_add_or_delete_listen_port_to_process0(qtuple->src_port, get_global_cfg_params()->process_idx, 1); + } + } + return 0; +@@ -197,7 +197,6 @@ int32_t vdev_reg_xmit(enum reg_ring_type type, struct gazelle_quintuple *qtuple) + return 0; + } + +- + int32_t ret; + uint32_t sent_pkts = 0; + void *free_buf[VDEV_REG_QUEUE_SZ]; +-- +2.23.0 + diff --git a/0232-update-lstack.Makefile.patch b/0232-update-lstack.Makefile.patch new file mode 100644 index 0000000..d3ff2e8 --- /dev/null +++ b/0232-update-lstack.Makefile.patch @@ -0,0 +1,30 @@ +From c7a8bc37c1aa0fcb33005e49e4b84e46478b2c3c Mon Sep 17 00:00:00 2001 +From: jiangheng12 +Date: Thu, 30 Mar 2023 11:29:33 +0800 +Subject: [PATCH] update lstack.Makefile + +--- + src/lstack/lstack.Makefile | 7 ++++++- + 1 file changed, 6 insertions(+), 1 deletion(-) + +diff --git a/src/lstack/lstack.Makefile b/src/lstack/lstack.Makefile +index dc98a1b..7da439d 100644 +--- a/src/lstack/lstack.Makefile ++++ b/src/lstack/lstack.Makefile +@@ -40,7 +40,12 @@ WRAP_API := epoll_ctl \ + close \ + ioctl \ + sigaction \ +- fork ++ fork \ ++ epoll_create1 \ ++ readv \ ++ writev \ ++ poll \ ++ ppoll + + WRAP_LDFLAGS = $(patsubst %, $(WRAP_PREFIX)%, $(WRAP_API)) + +-- +2.23.0 + diff --git a/0233-fix-gazellectl-x-error-when-multiplt-user-nic-config.patch b/0233-fix-gazellectl-x-error-when-multiplt-user-nic-config.patch new file mode 100644 index 0000000..3f7ca6e --- /dev/null +++ b/0233-fix-gazellectl-x-error-when-multiplt-user-nic-config.patch @@ -0,0 +1,68 @@ +From dcfd85d26a09a516a924350fa6404ec4237c2514 Mon Sep 17 00:00:00 2001 +From: jiangheng12 +Date: Fri, 31 Mar 2023 16:05:46 +0800 +Subject: [PATCH] fix gazellectl -x error when multiplt user nic config + +--- + src/lstack/core/lstack_dpdk.c | 3 ++- + src/lstack/core/lstack_stack_stat.c | 2 +- + src/lstack/include/lstack_dpdk.h | 2 +- + src/ltran/ltran_dfx.c | 2 +- + 4 files changed, 5 insertions(+), 4 deletions(-) + +diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c +index ebfebaa..e386dfc 100644 +--- a/src/lstack/core/lstack_dpdk.c ++++ b/src/lstack/core/lstack_dpdk.c +@@ -298,7 +298,8 @@ void lstack_log_level_init(void) + } + + // get port id +-inline uint16_t get_port_id(){ ++inline uint16_t get_port_id(void) ++{ + uint16_t port_id = get_global_cfg_params()->port_id; + return port_id; + } +diff --git a/src/lstack/core/lstack_stack_stat.c b/src/lstack/core/lstack_stack_stat.c +index eed7fbf..d1c749a 100644 +--- a/src/lstack/core/lstack_stack_stat.c ++++ b/src/lstack/core/lstack_stack_stat.c +@@ -260,7 +260,7 @@ int32_t handle_stack_cmd(int32_t fd, enum GAZELLE_STAT_MODE stat_mode) + struct protocol_stack_group *stack_group = get_protocol_stack_group(); + + if (stat_mode == GAZELLE_STAT_LSTACK_SHOW_XSTATS) { +- dpdk_nic_xstats_get(&dfx, 0); ++ dpdk_nic_xstats_get(&dfx, get_port_id()); + dfx.tid = 0; + dfx.eof = 1; + send_control_cmd_data(fd, &dfx); +diff --git a/src/lstack/include/lstack_dpdk.h b/src/lstack/include/lstack_dpdk.h +index 7c222ca..a896903 100644 +--- a/src/lstack/include/lstack_dpdk.h ++++ b/src/lstack/include/lstack_dpdk.h +@@ -52,7 +52,7 @@ void dpdk_skip_nic_init(void); + int32_t dpdk_init_lstack_kni(void); + void dpdk_restore_pci(void); + bool port_in_stack_queue(uint32_t src_ip, uint32_t dst_ip, uint16_t src_port, uint16_t dst_port); +-uint16_t get_port_id(); ++uint16_t get_port_id(void); + struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf,uint32_t mbuf_cache_size, uint16_t queue_id); + + void dpdk_nic_xstats_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id); +diff --git a/src/ltran/ltran_dfx.c b/src/ltran/ltran_dfx.c +index 9180e89..761bbb7 100644 +--- a/src/ltran/ltran_dfx.c ++++ b/src/ltran/ltran_dfx.c +@@ -137,7 +137,7 @@ static void gazelle_print_lstack_xstats(void *buf, const struct gazelle_stat_msg + struct nic_eth_xstats *xstats = &stat->data.nic_xstats; + static const char *nic_stats_border = "########################"; + +- printf("###### NIC extended statistics for port %-2d #########\n", 0); ++ printf("###### NIC extended statistics for port %-2d #########\n", xstats->port_id); + printf("%s############################\n",nic_stats_border); + if (xstats->len <= 0 || xstats->len > RTE_ETH_XSTATS_MAX_LEN) { + printf("xstats item(%d) num error!\n", xstats->len); +-- +2.23.0 + diff --git a/0234-fix-client-connect-number-unbalance-on-lstack.patch b/0234-fix-client-connect-number-unbalance-on-lstack.patch new file mode 100644 index 0000000..94b6c04 --- /dev/null +++ b/0234-fix-client-connect-number-unbalance-on-lstack.patch @@ -0,0 +1,93 @@ +From 3a1ff6aa0c9d85e3a5021fd3b6d788efe77db52a Mon Sep 17 00:00:00 2001 +From: jiangheng12 +Date: Fri, 31 Mar 2023 20:15:28 +0800 +Subject: [PATCH] fix client connect number unbalance on lstack + +--- + src/lstack/core/lstack_lwip.c | 1 - + src/lstack/core/lstack_protocol_stack.c | 8 +++++++- + src/lstack/include/lstack_protocol_stack.h | 1 + + 3 files changed, 8 insertions(+), 2 deletions(-) + +diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c +index 71abbc6..ebdb469 100644 +--- a/src/lstack/core/lstack_lwip.c ++++ b/src/lstack/core/lstack_lwip.c +@@ -192,7 +192,6 @@ void gazelle_init_sock(int32_t fd) + (void)replenish_send_idlembuf(stack, sock); + + sock->stack = stack; +- sock->stack->conn_num++; + init_list_node_null(&sock->recv_list); + init_list_node_null(&sock->event_list); + } +diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c +index 92300ef..76914f8 100644 +--- a/src/lstack/core/lstack_protocol_stack.c ++++ b/src/lstack/core/lstack_protocol_stack.c +@@ -124,6 +124,7 @@ struct protocol_stack *get_bind_protocol_stack(void) + + /* same app communication thread bind same stack */ + if (bind_stack) { ++ bind_stack->conn_num++; + return bind_stack; + } + +@@ -140,6 +141,7 @@ struct protocol_stack *get_bind_protocol_stack(void) + return NULL; + } + } else { ++ pthread_spin_lock(&stack_group->socket_lock); + for (uint16_t i = 0; i < stack_group->stack_num; i++) { + struct protocol_stack* stack = stack_group->stacks[i]; + if (get_global_cfg_params()->seperate_send_recv) { +@@ -147,7 +149,7 @@ struct protocol_stack *get_bind_protocol_stack(void) + index = i; + min_conn_num = stack->conn_num; + } +- }else { ++ } else { + if (stack->conn_num < min_conn_num) { + index = i; + min_conn_num = stack->conn_num; +@@ -156,7 +158,9 @@ struct protocol_stack *get_bind_protocol_stack(void) + } + } + ++ stack_group->stacks[index]->conn_num++; + bind_stack = stack_group->stacks[index]; ++ pthread_spin_unlock(&stack_group->socket_lock); + return stack_group->stacks[index]; + } + +@@ -541,6 +545,7 @@ int32_t init_protocol_stack(void) + + init_list_node(&stack_group->poll_list); + pthread_spin_init(&stack_group->poll_list_lock, PTHREAD_PROCESS_PRIVATE); ++ pthread_spin_init(&stack_group->socket_lock, PTHREAD_PROCESS_PRIVATE); + + + if (init_protocol_sem() != 0) { +@@ -689,6 +694,7 @@ void stack_accept(struct rpc_msg *msg) + } + + msg->result = accept_fd; ++ sock->stack->conn_num++; + if (rte_ring_count(sock->conn->recvmbox->ring)) { + add_recv_list(accept_fd); + } +diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h +index 001ab47..c75161f 100644 +--- a/src/lstack/include/lstack_protocol_stack.h ++++ b/src/lstack/include/lstack_protocol_stack.h +@@ -103,6 +103,7 @@ struct protocol_stack_group { + /* dfx stats */ + bool latency_start; + uint64_t call_alloc_fail; ++ pthread_spinlock_t socket_lock; + }; + + long get_stack_tid(void); +-- +2.23.0 + diff --git a/gazelle.spec b/gazelle.spec index 7087237..1a230aa 100644 --- a/gazelle.spec +++ b/gazelle.spec @@ -2,7 +2,7 @@ Name: gazelle Version: 1.0.1 -Release: 60 +Release: 61 Summary: gazelle is a high performance user-mode stack License: MulanPSL-2.0 URL: https://gitee.com/openeuler/gazelle @@ -244,6 +244,11 @@ Patch9226: 0226-optimite-select_path-and-pbuf_take.patch Patch9227: 0227-fix-build-err-on-select_path.patch Patch9228: 0228-set-kni_switch-valid-only-in-primary-process.patch Patch9229: 0229-add-socket-check-before-write-it.patch +Patch9230: 0230-optimize-do_close.patch +Patch9231: 0231-fix-config-flow-rule-race.patch +Patch9232: 0232-update-lstack.Makefile.patch +Patch9233: 0233-fix-gazellectl-x-error-when-multiplt-user-nic-config.patch +Patch9234: 0234-fix-client-connect-number-unbalance-on-lstack.patch %description %{name} is a high performance user-mode stack. @@ -284,6 +289,13 @@ install -Dpm 0640 %{_builddir}/%{name}-%{version}/src/ltran/ltran.conf %{b %config(noreplace) %{conf_path}/ltran.conf %changelog +* Fri Mar 31 2023 jiangheng12 - 1.0.1-61 +- fix gazellectl -x error when multiplt user nic config +- update lstack.Makefile +- fix config flow rule race +- optimize do_close +- fix client connect number unbalance on lstack + * Tue Mar 28 2023 jiangheng12 - 1.0.1-60 - add socket check before write it - set kni_switch valid only in primary process