Add feature for nfs client support multipath

Signed-off-by: mingqian218472 <zhangmingqian.zhang@huawei.com>

0001-nfs_add_api_to_support_enfs_registe_and_handle_mount_option.patch

@@ -0,0 +1,774 @@
+diff --git a/fs/nfs/client.c b/fs/nfs/client.c
+index 7d02dc52209d..50820a8a684a 100644
+--- a/fs/nfs/client.c
++++ b/fs/nfs/client.c
+@@ -48,7 +48,7 @@
+ #include "callback.h"
+ #include "delegation.h"
+ #include "iostat.h"
+-#include "internal.h"
++#include "enfs_adapter.h"
+ #include "fscache.h"
+ #include "pnfs.h"
+ #include "nfs.h"
+@@ -255,6 +255,7 @@ void nfs_free_client(struct nfs_client *clp)
+ 	put_nfs_version(clp->cl_nfs_mod);
+ 	kfree(clp->cl_hostname);
+ 	kfree(clp->cl_acceptor);
++	nfs_free_multi_path_client(clp);
+ 	kfree(clp);
+ }
+ EXPORT_SYMBOL_GPL(nfs_free_client);
+@@ -330,6 +331,9 @@ static struct nfs_client *nfs_match_client(const struct nfs_client_initdata *dat
+ 							   sap))
+ 				continue;
+ 
++		if (!nfs_multipath_client_match(clp, data))
++			continue;
++
+ 		refcount_inc(&clp->cl_count);
+ 		return clp;
+ 	}
+@@ -512,6 +516,9 @@ int nfs_create_rpc_client(struct nfs_client *clp,
+ 		.program	= &nfs_program,
+ 		.version	= clp->rpc_ops->version,
+ 		.authflavor	= flavor,
++#if IS_ENABLED(CONFIG_ENFS)
++		.multipath_option = cl_init->enfs_option,
++#endif
+ 	};
+ 
+ 	if (test_bit(NFS_CS_DISCRTRY, &clp->cl_flags))
+@@ -634,6 +641,13 @@ struct nfs_client *nfs_init_client(struct nfs_client *clp,
+ 	/* the client is already initialised */
+ 	if (clp->cl_cons_state == NFS_CS_READY)
+ 		return clp;
++	error = nfs_create_multi_path_client(clp, cl_init);
++	if (error < 0) {
++		dprintk("%s: create failed.%d!\n", __func__, error);
++		nfs_put_client(clp);
++		clp = ERR_PTR(error);
++		return clp;
++	}
+ 
+ 	/*
+ 	 * Create a client RPC handle for doing FSSTAT with UNIX auth only
+@@ -666,6 +680,9 @@ static int nfs_init_server(struct nfs_server *server,
+ 		.net = data->net,
+ 		.timeparms = &timeparms,
+ 		.init_flags = (1UL << NFS_CS_REUSEPORT),
++#if IS_ENABLED(CONFIG_ENFS)
++		.enfs_option = data->enfs_option,
++#endif
+ 	};
+ 	struct nfs_client *clp;
+ 	int error;
+diff --git a/fs/nfs/enfs_adapter.c b/fs/nfs/enfs_adapter.c
+new file mode 100644
+index 000000000000..7fde5b6fae90
+--- /dev/null
++++ b/fs/nfs/enfs_adapter.c
+@@ -0,0 +1,273 @@
++// SPDX-License-Identifier: GPL-2.0
++/*
++ * Client-side ENFS adapter.
++ *
++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved.
++ */
++#include <linux/types.h>
++#include <linux/sunrpc/clnt.h>
++#include <linux/nfs.h>
++#include <linux/nfs4.h>
++#include <linux/nfs3.h>
++#include <linux/nfs_fs.h>
++#include <linux/nfs_fs_sb.h>
++#include <linux/sunrpc/sched.h>
++#include <linux/nfs_iostat.h>
++#include "enfs_adapter.h"
++#include "iostat.h"
++
++struct enfs_adapter_ops __rcu *enfs_adapter;
++
++int enfs_adapter_register(struct enfs_adapter_ops *ops)
++{
++	struct enfs_adapter_ops *old;
++
++	old = cmpxchg((struct enfs_adapter_ops **)&enfs_adapter, NULL, ops);
++	if (old == NULL || old == ops)
++		return 0;
++	pr_err("regist %s ops %p failed. old %p\n", __func__, ops, old);
++	return -EPERM;
++}
++EXPORT_SYMBOL_GPL(enfs_adapter_register);
++
++int enfs_adapter_unregister(struct enfs_adapter_ops *ops)
++{
++	struct enfs_adapter_ops *old;
++
++	old = cmpxchg((struct enfs_adapter_ops **)&enfs_adapter, ops, NULL);
++	if (old == ops || old == NULL)
++		return 0;
++	pr_err("unregist %s ops %p failed. old %p\n", __func__, ops, old);
++	return -EPERM;
++}
++EXPORT_SYMBOL_GPL(enfs_adapter_unregister);
++
++struct enfs_adapter_ops *nfs_multipath_router_get(void)
++{
++	struct enfs_adapter_ops *ops;
++
++	rcu_read_lock();
++	ops = rcu_dereference(enfs_adapter);
++	if (ops == NULL) {
++		rcu_read_unlock();
++		return NULL;
++	}
++	if (!try_module_get(ops->owner))
++		ops = NULL;
++	rcu_read_unlock();
++	return ops;
++}
++
++void nfs_multipath_router_put(struct enfs_adapter_ops *ops)
++{
++	if (ops)
++		module_put(ops->owner);
++}
++
++bool is_valid_option(enum nfs_multi_path_options option)
++{
++	if (option < REMOTEADDR || option >= INVALID_OPTION) {
++		pr_warn("%s: ENFS invalid option %d\n", __func__, option);
++		return false;
++	}
++
++	return true;
++}
++
++int enfs_parse_mount_options(enum nfs_multi_path_options option, char *str,
++			struct nfs_parsed_mount_data *mnt)
++{
++	int rc;
++	struct enfs_adapter_ops *ops;
++
++	ops = nfs_multipath_router_get();
++	if ((ops == NULL) || (ops->parse_mount_options == NULL) ||
++		!is_valid_option(option)) {
++		nfs_multipath_router_put(ops);
++		dfprintk(MOUNT,
++			"NFS: parsing nfs mount option enfs not load[%s]\n"
++		,  __func__);
++		return -EOPNOTSUPP;
++	}
++    // nfs_multipath_parse_options
++	dfprintk(MOUNT, "NFS:   parsing nfs mount option '%s' type: %d[%s]\n"
++			, str, option, __func__);
++	rc = ops->parse_mount_options(option, str, &mnt->enfs_option, mnt->net);
++	nfs_multipath_router_put(ops);
++	return rc;
++}
++
++void enfs_free_mount_options(struct nfs_parsed_mount_data *data)
++{
++	struct enfs_adapter_ops *ops;
++
++	if (data->enfs_option == NULL)
++		return;
++
++	ops = nfs_multipath_router_get();
++	if ((ops == NULL) || (ops->free_mount_options == NULL)) {
++		nfs_multipath_router_put(ops);
++		return;
++	}
++	ops->free_mount_options((void *)&data->enfs_option);
++	nfs_multipath_router_put(ops);
++}
++
++int nfs_create_multi_path_client(struct nfs_client *client,
++			const struct nfs_client_initdata *cl_init)
++{
++	int ret = 0;
++	struct enfs_adapter_ops *ops;
++
++	if (cl_init->enfs_option == NULL)
++		return 0;
++
++	ops = nfs_multipath_router_get();
++	if (ops != NULL && ops->client_info_init != NULL)
++		ret = ops->client_info_init(
++			(void *)&client->cl_multipath_data, cl_init);
++	nfs_multipath_router_put(ops);
++
++	return ret;
++}
++EXPORT_SYMBOL_GPL(nfs_create_multi_path_client);
++
++void nfs_free_multi_path_client(struct nfs_client *clp)
++{
++	struct enfs_adapter_ops *ops;
++
++	if (clp->cl_multipath_data == NULL)
++		return;
++
++	ops = nfs_multipath_router_get();
++	if (ops != NULL && ops->client_info_free != NULL)
++		ops->client_info_free(clp->cl_multipath_data);
++	nfs_multipath_router_put(ops);
++}
++
++int nfs_multipath_client_match(struct nfs_client *clp,
++			const struct nfs_client_initdata *sap)
++{
++	bool ret = true;
++	struct enfs_adapter_ops *ops;
++
++	pr_info("%s src %p dst %p\n.", __func__,
++	clp->cl_multipath_data, sap->enfs_option);
++
++	if (clp->cl_multipath_data == NULL && sap->enfs_option == NULL)
++		return true;
++
++	if ((clp->cl_multipath_data == NULL && sap->enfs_option) ||
++		(clp->cl_multipath_data && sap->enfs_option == NULL)) {
++		pr_err("not match client src %p dst %p\n.",
++				clp->cl_multipath_data, sap->enfs_option);
++		return false;
++	}
++
++	ops = nfs_multipath_router_get();
++	if (ops != NULL && ops->client_info_match != NULL)
++		ret = ops->client_info_match(clp->cl_multipath_data,
++					sap->enfs_option);
++	nfs_multipath_router_put(ops);
++
++	return ret;
++}
++
++int nfs4_multipath_client_match(struct nfs_client *src, struct nfs_client *dst)
++{
++	int ret = true;
++	struct enfs_adapter_ops *ops;
++
++	if (src->cl_multipath_data == NULL && dst->cl_multipath_data == NULL)
++		return true;
++
++	if (src->cl_multipath_data == NULL || dst->cl_multipath_data == NULL)
++		return false;
++
++	ops = nfs_multipath_router_get();
++	if (ops != NULL && ops->nfs4_client_info_match != NULL)
++		ret = ops->nfs4_client_info_match(src->cl_multipath_data,
++					src->cl_multipath_data);
++	nfs_multipath_router_put(ops);
++
++	return ret;
++}
++EXPORT_SYMBOL_GPL(nfs4_multipath_client_match);
++
++void nfs_multipath_show_client_info(struct seq_file *mount_option,
++			struct nfs_server *server)
++{
++	struct enfs_adapter_ops *ops;
++
++	if (mount_option == NULL || server == NULL ||
++		server->client == NULL ||
++		server->nfs_client->cl_multipath_data == NULL)
++		return;
++
++	ops = nfs_multipath_router_get();
++	if (ops != NULL && ops->client_info_show != NULL)
++		ops->client_info_show(mount_option, server);
++	nfs_multipath_router_put(ops);
++}
++
++int nfs_remount_iplist(struct nfs_client *nfs_client, void *data)
++{
++	int ret = 0;
++	struct enfs_adapter_ops *ops;
++	struct nfs_parsed_mount_data *parsed_data = (struct nfs_parsed_mount_data *)data;
++
++	if (!parsed_data->enfs_option)
++		return 0;
++
++	if (nfs_client == NULL || nfs_client->cl_rpcclient == NULL)
++		return 0;
++
++	ops = nfs_multipath_router_get();
++	if (ops != NULL && ops->remount_ip_list != NULL)
++		ret = ops->remount_ip_list(nfs_client, parsed_data->enfs_option);
++	nfs_multipath_router_put(ops);
++	return ret;
++}
++EXPORT_SYMBOL_GPL(nfs_remount_iplist);
++
++/*
++ * Error-check and convert a string of mount options from
++ * user space into a data structure. The whole mount string
++ * is processed; bad options are skipped as they are encountered.
++ * If there were no errors, return 1; otherwise return zero(0)
++ */
++int enfs_check_mount_parse_info(char *p, int token,
++		struct nfs_parsed_mount_data *mnt, const substring_t *args)
++{
++	char *string;
++	int rc;
++	string = match_strdup(args);
++	if (string == NULL) {
++		printk(KERN_INFO "NFS: not enough memory to parse option\n");
++		return 0;
++	}
++	rc = enfs_parse_mount_options(get_nfs_multi_path_opt(token),
++					string, mnt);
++
++	kfree(string);
++	switch (rc) {
++	case  0:
++		return 1;
++	case -ENOMEM:
++		printk(KERN_INFO "NFS: not enough memory to parse option\n");
++		return 0;
++	case -ENOSPC:
++		printk(KERN_INFO "NFS: param is more than supported limit: %d\n", rc);
++		return 0;
++	case -EINVAL:
++		printk(KERN_INFO "NFS: bad IP address specified: %s\n", p);
++		return 0;
++	case -ENOTSUPP:
++		printk(KERN_INFO "NFS: bad IP address specified: %s\n", p);
++		return 0;
++	case -EOPNOTSUPP:
++		printk(KERN_INFO "NFS: bad IP address specified: %s\n", p);
++		return 0;
++	}
++	return 1;
++}
+diff --git a/fs/nfs/enfs_adapter.h b/fs/nfs/enfs_adapter.h
+new file mode 100644
+index 000000000000..c7483ca1222a
+--- /dev/null
++++ b/fs/nfs/enfs_adapter.h
+@@ -0,0 +1,116 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ *  Client-side ENFS adapt header.
++ *
++ *  Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved.
++ */
++#ifndef _NFS_MULTIPATH_H_
++#define _NFS_MULTIPATH_H_
++
++#include <linux/parser.h>
++#include "internal.h"
++
++#if IS_ENABLED(CONFIG_ENFS)
++enum nfs_multi_path_options {
++	REMOTEADDR,
++	LOCALADDR,
++	REMOTEDNSNAME,
++	REMOUNTREMOTEADDR,
++	REMOUNTLOCALADDR,
++	INVALID_OPTION
++};
++
++
++struct enfs_adapter_ops {
++	const char *name;
++	struct module *owner;
++	int (*parse_mount_options)(enum nfs_multi_path_options option,
++		char *str, void **enfs_option, struct net *net_ns);
++
++	void (*free_mount_options)(void **data);
++
++	int (*client_info_init)(void **data,
++				const struct nfs_client_initdata *cl_init);
++	void (*client_info_free)(void *data);
++	int (*client_info_match)(void *src, void *dst);
++	int (*nfs4_client_info_match)(void *src, void *dst);
++	void (*client_info_show)(struct seq_file *mount_option, void *data);
++	int (*remount_ip_list)(struct nfs_client *nfs_client,
++						void *enfs_option);
++};
++
++int enfs_parse_mount_options(enum nfs_multi_path_options option, char *str,
++			struct nfs_parsed_mount_data *mnt);
++void enfs_free_mount_options(struct nfs_parsed_mount_data *data);
++int nfs_create_multi_path_client(struct nfs_client *client,
++			const struct nfs_client_initdata *cl_init);
++void nfs_free_multi_path_client(struct nfs_client *clp);
++int nfs_multipath_client_match(struct nfs_client *clp,
++			const struct nfs_client_initdata *sap);
++int nfs4_multipath_client_match(struct nfs_client *src, struct nfs_client *dst);
++void nfs_multipath_show_client_info(struct seq_file *mount_option,
++			struct nfs_server *server);
++int enfs_adapter_register(struct enfs_adapter_ops *ops);
++int enfs_adapter_unregister(struct enfs_adapter_ops *ops);
++int nfs_remount_iplist(struct nfs_client *nfs_client, void *data);
++int nfs4_create_multi_path(struct nfs_server *server,
++	struct nfs_parsed_mount_data *data,
++	const struct rpc_timeout *timeparms);
++int enfs_check_mount_parse_info(char *p, int token,
++		struct nfs_parsed_mount_data *mnt, const substring_t *args);
++
++#else
++static inline
++void nfs_free_multi_path_client(struct nfs_client *clp)
++{
++
++}
++
++static inline
++int nfs_multipath_client_match(struct nfs_client *clp,
++			const struct nfs_client_initdata *sap)
++{
++	return 1;
++}
++
++static inline
++int nfs_create_multi_path_client(struct nfs_client *client,
++			const struct nfs_client_initdata *cl_init)
++{
++	return 0;
++}
++
++static inline
++void nfs_multipath_show_client_info(struct seq_file *mount_option,
++			struct nfs_server *server)
++{
++
++}
++
++static inline
++int nfs4_multipath_client_match(struct nfs_client *src,
++			struct nfs_client *dst)
++{
++	return 1;
++}
++
++static inline
++void enfs_free_mount_options(struct nfs_parsed_mount_data *data)
++{
++
++}
++
++static inline
++int enfs_check_mount_parse_info(char *p, int token,
++		struct nfs_parsed_mount_data *mnt, const substring_t *args)
++{
++	return 1;
++}
++
++static inline
++int nfs_remount_iplist(struct nfs_client *nfs_client, void *data)
++{
++	return 0;
++}
++#endif // CONFIG_ENFS
++#endif // _NFS_MULTIPATH_H_
+diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
+index 0ce5a90640c4..84ac82dbb300 100644
+--- a/fs/nfs/internal.h
++++ b/fs/nfs/internal.h
+@@ -93,6 +93,9 @@ struct nfs_client_initdata {
+ 	u32 minorversion;
+ 	struct net *net;
+ 	const struct rpc_timeout *timeparms;
++#if IS_ENABLED(CONFIG_ENFS)
++	void *enfs_option; /* struct multipath_mount_options * */
++#endif
+ };
+ 
+ /*
+@@ -135,6 +138,9 @@ struct nfs_parsed_mount_data {
+ 
+ 	struct security_mnt_opts lsm_opts;
+ 	struct net		*net;
++#if IS_ENABLED(CONFIG_ENFS)
++	void *enfs_option; /* struct multipath_mount_options * */
++#endif
+ };
+ 
+ /* mount_clnt.c */
+@@ -430,6 +436,10 @@ extern void nfs_sb_deactive(struct super_block *sb);
+ extern int nfs_client_for_each_server(struct nfs_client *clp,
+ 				      int (*fn)(struct nfs_server *, void *),
+ 				      void *data);
++#if IS_ENABLED(CONFIG_ENFS)
++extern enum nfs_multi_path_options get_nfs_multi_path_opt(int token);
++#endif
++
+ /* io.c */
+ extern void nfs_start_io_read(struct inode *inode);
+ extern void nfs_end_io_read(struct inode *inode);
+diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c
+index 1350ea673672..f97646b9882e 100644
+--- a/fs/nfs/nfs4client.c
++++ b/fs/nfs/nfs4client.c
+@@ -10,7 +10,7 @@
+ #include <linux/sunrpc/xprt.h>
+ #include <linux/sunrpc/bc_xprt.h>
+ #include <linux/sunrpc/rpc_pipe_fs.h>
+-#include "internal.h"
++#include "enfs_adapter.h"
+ #include "callback.h"
+ #include "delegation.h"
+ #include "nfs4session.h"
+@@ -225,6 +225,14 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init)
+ 	__set_bit(NFS_CS_DISCRTRY, &clp->cl_flags);
+ 	__set_bit(NFS_CS_NO_RETRANS_TIMEOUT, &clp->cl_flags);
+ 
++	err = nfs_create_multi_path_client(clp, cl_init);
++	if (err < 0) {
++		dprintk("%s: create failed.%d\n", __func__, err);
++		nfs_put_client(clp);
++		clp = ERR_PTR(err);
++		return clp;
++	}
++
+ 	/*
+ 	 * Set up the connection to the server before we add add to the
+ 	 * global list.
+@@ -529,6 +537,9 @@ static int nfs4_match_client(struct nfs_client  *pos,  struct nfs_client *new,
+ 	if (!nfs4_match_client_owner_id(pos, new))
+ 		return 1;
+ 
++	if (!nfs4_multipath_client_match(pos, new))
++		return 1;
++
+ 	return 0;
+ }
+ 
+@@ -860,7 +871,7 @@ static int nfs4_set_client(struct nfs_server *server,
+ 		const size_t addrlen,
+ 		const char *ip_addr,
+ 		int proto, const struct rpc_timeout *timeparms,
+-		u32 minorversion, struct net *net)
++		u32 minorversion, struct net *net, void *enfs_option)
+ {
+ 	struct nfs_client_initdata cl_init = {
+ 		.hostname = hostname,
+@@ -872,6 +883,9 @@ static int nfs4_set_client(struct nfs_server *server,
+ 		.minorversion = minorversion,
+ 		.net = net,
+ 		.timeparms = timeparms,
++#if IS_ENABLED(CONFIG_ENFS)
++		.enfs_option = enfs_option,
++#endif
+ 	};
+ 	struct nfs_client *clp;
+ 
+@@ -1050,6 +1064,7 @@ static int nfs4_init_server(struct nfs_server *server,
+ {
+ 	struct rpc_timeout timeparms;
+ 	int error;
++	void *enfs_option = NULL;
+ 
+ 	nfs_init_timeout_values(&timeparms, data->nfs_server.protocol,
+ 			data->timeo, data->retrans);
+@@ -1067,6 +1082,10 @@ static int nfs4_init_server(struct nfs_server *server,
+ 	else
+ 		data->selected_flavor = RPC_AUTH_UNIX;
+ 
++#if IS_ENABLED(CONFIG_ENFS)
++	enfs_option = data->enfs_option;
++#endif
++
+ 	/* Get a client record */
+ 	error = nfs4_set_client(server,
+ 			data->nfs_server.hostname,
+@@ -1076,7 +1095,7 @@ static int nfs4_init_server(struct nfs_server *server,
+ 			data->nfs_server.protocol,
+ 			&timeparms,
+ 			data->minorversion,
+-			data->net);
++			data->net, enfs_option);
+ 	if (error < 0)
+ 		return error;
+ 
+@@ -1161,7 +1180,7 @@ struct nfs_server *nfs4_create_referral_server(struct nfs_clone_mount *data,
+ 				XPRT_TRANSPORT_RDMA,
+ 				parent_server->client->cl_timeout,
+ 				parent_client->cl_mvops->minor_version,
+-				parent_client->cl_net);
++				parent_client->cl_net, NULL);
+ 	if (!error)
+ 		goto init_server;
+ #endif	/* IS_ENABLED(CONFIG_SUNRPC_XPRT_RDMA) */
+@@ -1174,7 +1193,7 @@ struct nfs_server *nfs4_create_referral_server(struct nfs_clone_mount *data,
+ 				XPRT_TRANSPORT_TCP,
+ 				parent_server->client->cl_timeout,
+ 				parent_client->cl_mvops->minor_version,
+-				parent_client->cl_net);
++				parent_client->cl_net, NULL);
+ 	if (error < 0)
+ 		goto error;
+ 
+@@ -1269,7 +1288,7 @@ int nfs4_update_server(struct nfs_server *server, const char *hostname,
+ 	set_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status);
+ 	error = nfs4_set_client(server, hostname, sap, salen, buf,
+ 				clp->cl_proto, clnt->cl_timeout,
+-				clp->cl_minorversion, net);
++				clp->cl_minorversion, net, NULL);
+ 	clear_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status);
+ 	if (error != 0) {
+ 		nfs_server_insert_lists(server);
+diff --git a/fs/nfs/super.c b/fs/nfs/super.c
+index a05e1eb2c3fd..f26bdaa45fca 100644
+--- a/fs/nfs/super.c
++++ b/fs/nfs/super.c
+@@ -51,7 +51,6 @@
+ #include <linux/netdevice.h>
+ #include <linux/nfs_xdr.h>
+ #include <linux/magic.h>
+-#include <linux/parser.h>
+ #include <linux/nsproxy.h>
+ #include <linux/rcupdate.h>
+ 
+@@ -61,7 +60,7 @@
+ #include "callback.h"
+ #include "delegation.h"
+ #include "iostat.h"
+-#include "internal.h"
++#include "enfs_adapter.h"
+ #include "fscache.h"
+ #include "nfs4session.h"
+ #include "pnfs.h"
+@@ -113,6 +112,12 @@ enum {
+ 
+ 	/* Special mount options */
+ 	Opt_userspace, Opt_deprecated, Opt_sloppy,
++#if IS_ENABLED(CONFIG_ENFS)
++	Opt_remote_iplist,
++	Opt_local_iplist,
++	Opt_remote_dnslist,
++	Opt_enfs_info,
++#endif
+ 
+ 	Opt_err
+ };
+@@ -183,6 +188,13 @@ static const match_table_t nfs_mount_option_tokens = {
+ 	{ Opt_fscache_uniq, "fsc=%s" },
+ 	{ Opt_local_lock, "local_lock=%s" },
+ 
++#if IS_ENABLED(CONFIG_ENFS)
++	{ Opt_remote_iplist, "remoteaddrs=%s" },
++	{ Opt_local_iplist, "localaddrs=%s" },
++	{ Opt_remote_dnslist, "remotednsname=%s" },
++	{ Opt_enfs_info, "enfs_info=%s" },
++#endif
++
+ 	/* The following needs to be listed after all other options */
+ 	{ Opt_nfsvers, "v%s" },
+ 
+@@ -365,6 +377,21 @@ static struct shrinker acl_shrinker = {
+ 	.seeks		= DEFAULT_SEEKS,
+ };
+ 
++#if IS_ENABLED(CONFIG_ENFS)
++enum nfs_multi_path_options get_nfs_multi_path_opt(int token)
++{
++	switch (token) {
++	case Opt_remote_iplist:
++		return REMOUNTREMOTEADDR;
++	case Opt_local_iplist:
++		return REMOUNTLOCALADDR;
++	case Opt_remote_dnslist:
++		return REMOTEDNSNAME;
++	}
++	return INVALID_OPTION;
++}
++#endif
++
+ /*
+  * Register the NFS filesystems
+  */
+@@ -758,6 +785,9 @@ int nfs_show_options(struct seq_file *m, struct dentry *root)
+ 	seq_printf(m, ",addr=%s",
+ 			rpc_peeraddr2str(nfss->nfs_client->cl_rpcclient,
+ 							RPC_DISPLAY_ADDR));
++
++	nfs_multipath_show_client_info(m, nfss);
++
+ 	rcu_read_unlock();
+ 
+ 	return 0;
+@@ -853,6 +883,8 @@ int nfs_show_stats(struct seq_file *m, struct dentry *root)
+ 	seq_puts(m, root->d_sb->s_flags & SB_NODIRATIME ? ",nodiratime" : "");
+ 	nfs_show_mount_options(m, nfss, 1);
+ 
++	nfs_multipath_show_client_info(m, nfss);
++
+ 	seq_printf(m, "\n\tage:\t%lu", (jiffies - nfss->mount_time) / HZ);
+ 
+ 	show_implementation_id(m, nfss);
+@@ -977,6 +1009,7 @@ static void nfs_free_parsed_mount_data(struct nfs_parsed_mount_data *data)
+ 		kfree(data->nfs_server.export_path);
+ 		kfree(data->nfs_server.hostname);
+ 		kfree(data->fscache_uniq);
++		enfs_free_mount_options(data);
+ 		security_free_mnt_opts(&data->lsm_opts);
+ 		kfree(data);
+ 	}
+@@ -1641,7 +1674,6 @@ static int nfs_parse_mount_options(char *raw,
+ 				return 0;
+ 			};
+ 			break;
+-
+ 		/*
+ 		 * Special options
+ 		 */
+@@ -1654,7 +1686,18 @@ static int nfs_parse_mount_options(char *raw,
+ 			dfprintk(MOUNT, "NFS:   ignoring mount option "
+ 					"'%s'\n", p);
+ 			break;
+-
++#if IS_ENABLED(CONFIG_ENFS)
++		case Opt_remote_iplist:
++		case Opt_local_iplist:
++		case Opt_remote_dnslist:
++			rc = enfs_check_mount_parse_info(p,
++					token, mnt, args);
++			if (rc != 1) 
++				return rc;
++			break;
++		case Opt_enfs_info:
++			break;
++#endif
+ 		default:
+ 			invalid_option = 1;
+ 			dfprintk(MOUNT, "NFS:   unrecognized mount option "
+@@ -2335,6 +2378,10 @@ nfs_remount(struct super_block *sb, int *flags, char *raw_data)
+ 	if (!nfs_parse_mount_options((char *)options, data))
+ 		goto out;
+ 
++	error = nfs_remount_iplist(nfss->nfs_client, data);
++	if (error)
++		goto out;
++
+ 	/*
+ 	 * noac is a special case. It implies -o sync, but that's not
+ 	 * necessarily reflected in the mtab options. do_remount_sb
+@@ -2347,6 +2394,8 @@ nfs_remount(struct super_block *sb, int *flags, char *raw_data)
+ 	/* compare new mount options with old ones */
+ 	error = nfs_compare_remount_data(nfss, data);
+ out:
++	/* release remount option member */
++	enfs_free_mount_options(data);
+ 	nfs_free_parsed_mount_data(data);
+ 	return error;
+ }
+diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
+index 7023ae64e3d7..2c19678afe8d 100644
+--- a/include/linux/nfs_fs_sb.h
++++ b/include/linux/nfs_fs_sb.h
+@@ -123,6 +123,11 @@ struct nfs_client {
+ 
+ 	struct net		*cl_net;
+ 	struct list_head	pending_cb_stateids;
++
++#if IS_ENABLED(CONFIG_ENFS)
++	/* multi path private structure (struct multipath_client_info *) */
++	void *cl_multipath_data;
++#endif
+ };
+ 
+ /*

0002-sunrpc_add_api_to_support_enfs_registe_and_create_multipath_then_dispatch_IO.patch

@@ -0,0 +1,799 @@
+diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h
+index 8aa865bce4f6..89178f78de8c 100644
+--- a/include/linux/sunrpc/clnt.h
++++ b/include/linux/sunrpc/clnt.h
+@@ -70,6 +70,10 @@ struct rpc_clnt {
+ 	struct dentry		*cl_debugfs;	/* debugfs directory */
+ #endif
+ 	struct rpc_xprt_iter	cl_xpi;
++
++#if IS_ENABLED(CONFIG_ENFS)
++	bool cl_enfs;
++#endif
+ };
+ 
+ /*
+@@ -124,6 +128,9 @@ struct rpc_create_args {
+ 	unsigned long		flags;
+ 	char			*client_name;
+ 	struct svc_xprt		*bc_xprt;	/* NFSv4.1 backchannel */
++#if IS_ENABLED(CONFIG_ENFS)
++	void *multipath_option;
++#endif
+ };
+ 
+ struct rpc_add_xprt_test {
+@@ -221,6 +228,12 @@ bool rpc_clnt_xprt_switch_has_addr(struct rpc_clnt *clnt,
+ 			const struct sockaddr *sap);
+ void rpc_cleanup_clids(void);
+ 
++#if IS_ENABLED(CONFIG_ENFS)
++int
++rpc_clnt_test_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt,
++		   const struct rpc_call_ops *ops, void *data, int flags);
++#endif /* CONFIG_ENFS */
++
+ static inline int rpc_reply_expected(struct rpc_task *task)
+ {
+ 	return (task->tk_msg.rpc_proc != NULL) &&
+diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h
+index ad2e243f3f03..124f5a0faf3e 100644
+--- a/include/linux/sunrpc/sched.h
++++ b/include/linux/sunrpc/sched.h
+@@ -90,6 +90,9 @@ struct rpc_task {
+ 				tk_garb_retry : 2,
+ 				tk_cred_retry : 2,
+ 				tk_rebind_retry : 2;
++#if IS_ENABLED(CONFIG_ENFS)
++	unsigned long		tk_major_timeo;	/* major timeout ticks */
++#endif
+ };
+ 
+ typedef void			(*rpc_action)(struct rpc_task *);
+@@ -118,6 +121,9 @@ struct rpc_task_setup {
+  */
+ #define RPC_TASK_ASYNC		0x0001		/* is an async task */
+ #define RPC_TASK_SWAPPER	0x0002		/* is swapping in/out */
++#if IS_ENABLED(CONFIG_ENFS)
++#define RPC_TASK_FIXED	0x0004		/* detect xprt status task */
++#endif
+ #define RPC_CALL_MAJORSEEN	0x0020		/* major timeout seen */
+ #define RPC_TASK_ROOTCREDS	0x0040		/* force root creds */
+ #define RPC_TASK_DYNAMIC	0x0080		/* task was kmalloc'ed */
+@@ -257,6 +263,9 @@ void		rpc_destroy_mempool(void);
+ extern struct workqueue_struct *rpciod_workqueue;
+ extern struct workqueue_struct *xprtiod_workqueue;
+ void		rpc_prepare_task(struct rpc_task *task);
++#if IS_ENABLED(CONFIG_ENFS)
++void rpc_init_task_retry_counters(struct rpc_task *task);
++#endif
+ 
+ static inline int rpc_wait_for_completion_task(struct rpc_task *task)
+ {
+diff --git a/include/linux/sunrpc/sunrpc_enfs_adapter.h b/include/linux/sunrpc/sunrpc_enfs_adapter.h
+new file mode 100644
+index 000000000000..28abedcf5cf6
+--- /dev/null
++++ b/include/linux/sunrpc/sunrpc_enfs_adapter.h
+@@ -0,0 +1,128 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/* Client-side SUNRPC ENFS adapter header.
++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved.
++ */
++#ifndef _SUNRPC_ENFS_ADAPTER_H_
++#define _SUNRPC_ENFS_ADAPTER_H_
++#include <linux/sunrpc/clnt.h>
++
++#if IS_ENABLED(CONFIG_ENFS)
++
++static inline void rpc_xps_nactive_add_one(struct rpc_xprt_switch *xps)
++{
++	xps->xps_nactive--;
++}
++
++static inline void rpc_xps_nactive_sub_one(struct rpc_xprt_switch *xps)
++{
++	xps->xps_nactive--;
++}
++
++struct rpc_xprt	*rpc_task_get_xprt
++(struct rpc_clnt *clnt, struct rpc_xprt *xprt);
++
++struct rpc_multipath_ops {
++	struct module *owner;
++	void (*create_clnt)(struct rpc_create_args *args,
++			    struct rpc_clnt *clnt);
++	void (*releas_clnt)(struct rpc_clnt *clnt);
++	void (*create_xprt)(struct rpc_xprt *xprt);
++	void (*destroy_xprt)(struct rpc_xprt *xprt);
++	void (*xprt_iostat)(struct rpc_task *task);
++	void (*failover_handle)(struct rpc_task *task);
++	bool (*task_need_call_start_again)(struct rpc_task *task);
++	void (*adjust_task_timeout)(struct rpc_task *task, void *condition);
++	void (*init_task_req)(struct rpc_task *task, struct rpc_rqst *req);
++	bool (*prepare_transmit)(struct rpc_task *task);
++};
++
++extern struct rpc_multipath_ops __rcu *multipath_ops;
++void rpc_init_task_retry_counters(struct rpc_task *task);
++int rpc_multipath_ops_register(struct rpc_multipath_ops *ops);
++int rpc_multipath_ops_unregister(struct rpc_multipath_ops *ops);
++struct rpc_multipath_ops *rpc_multipath_ops_get(void);
++void rpc_multipath_ops_put(struct rpc_multipath_ops *ops);
++void rpc_task_release_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt);
++void rpc_multipath_ops_create_clnt(struct rpc_create_args *args,
++				   struct rpc_clnt *clnt);
++void rpc_multipath_ops_releas_clnt(struct rpc_clnt *clnt);
++bool rpc_multipath_ops_create_xprt(struct rpc_xprt *xprt);
++void rpc_multipath_ops_destroy_xprt(struct rpc_xprt *xprt);
++void rpc_multipath_ops_xprt_iostat(struct rpc_task *task);
++void rpc_multipath_ops_failover_handle(struct rpc_task *task);
++bool rpc_multipath_ops_task_need_call_start_again(struct rpc_task *task);
++void rpc_multipath_ops_adjust_task_timeout(struct rpc_task *task,
++					   void *condition);
++void rpc_multipath_ops_init_task_req(struct rpc_task *task,
++				     struct rpc_rqst *req);
++bool rpc_multipath_ops_prepare_transmit(struct rpc_task *task);
++
++#else
++static inline struct rpc_xprt *rpc_task_get_xprt(struct rpc_clnt *clnt,
++						 struct rpc_xprt *xprt)
++{
++	return NULL;
++}
++
++static inline void rpc_task_release_xprt(struct rpc_clnt *clnt,
++					 struct rpc_xprt *xprt)
++{
++}
++
++static inline void rpc_xps_nactive_add_one(struct rpc_xprt_switch *xps)
++{
++}
++
++static inline void rpc_xps_nactive_sub_one(struct rpc_xprt_switch *xps)
++{
++}
++
++static inline void rpc_multipath_ops_create_clnt
++(struct rpc_create_args *args, struct rpc_clnt *clnt)
++{
++}
++
++static inline void rpc_multipath_ops_releas_clnt(struct rpc_clnt *clnt)
++{
++}
++
++static inline bool rpc_multipath_ops_create_xprt(struct rpc_xprt *xprt)
++{
++	return false;
++}
++
++static inline void rpc_multipath_ops_destroy_xprt(struct rpc_xprt *xprt)
++{
++}
++
++static inline void rpc_multipath_ops_xprt_iostat(struct rpc_task *task)
++{
++}
++
++static inline void rpc_multipath_ops_failover_handle(struct rpc_task *task)
++{
++}
++
++static inline
++bool rpc_multipath_ops_task_need_call_start_again(struct rpc_task *task)
++{
++	return false;
++}
++
++static inline void
++rpc_multipath_ops_adjust_task_timeout(struct rpc_task *task, void *condition)
++{
++}
++
++static inline void
++rpc_multipath_ops_init_task_req(struct rpc_task *task, struct rpc_rqst *req)
++{
++}
++
++static inline bool rpc_multipath_ops_prepare_transmit(struct rpc_task *task)
++{
++	return false;
++}
++
++#endif
++#endif // _SUNRPC_ENFS_ADAPTER_H_
+diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
+index ccfacca1eba9..2e47b3577947 100644
+--- a/include/linux/sunrpc/xprt.h
++++ b/include/linux/sunrpc/xprt.h
+@@ -279,6 +279,10 @@ struct rpc_xprt {
+ 	atomic_t		inject_disconnect;
+ #endif
+ 	struct rcu_head		rcu;
++#if IS_ENABLED(CONFIG_ENFS)
++	atomic_long_t	queuelen;
++	void *multipath_context;
++#endif
+ };
+ 
+ #if defined(CONFIG_SUNRPC_BACKCHANNEL)
+diff --git a/include/linux/sunrpc/xprtmultipath.h b/include/linux/sunrpc/xprtmultipath.h
+index af1257c030d2..d54e4dbbbf34 100644
+--- a/include/linux/sunrpc/xprtmultipath.h
++++ b/include/linux/sunrpc/xprtmultipath.h
+@@ -22,6 +22,10 @@ struct rpc_xprt_switch {
+ 	const struct rpc_xprt_iter_ops *xps_iter_ops;
+ 
+ 	struct rcu_head		xps_rcu;
++#if IS_ENABLED(CONFIG_ENFS)
++	unsigned int		xps_nactive;
++	atomic_long_t		xps_queuelen;
++#endif
+ };
+ 
+ struct rpc_xprt_iter {
+@@ -69,4 +73,8 @@ extern struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi);
+ 
+ extern bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps,
+ 		const struct sockaddr *sap);
++#if IS_ENABLED(CONFIG_ENFS)
++extern void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
++		struct rpc_xprt *xprt);
++#endif
+ #endif
+diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
+index 0fc540b0d183..5ad3f18eb1c0 100644
+--- a/net/sunrpc/clnt.c
++++ b/net/sunrpc/clnt.c
+@@ -37,6 +37,7 @@
+ #include <linux/sunrpc/rpc_pipe_fs.h>
+ #include <linux/sunrpc/metrics.h>
+ #include <linux/sunrpc/bc_xprt.h>
++#include <linux/sunrpc/sunrpc_enfs_adapter.h>
+ #include <trace/events/sunrpc.h>
+ 
+ #include "sunrpc.h"
+@@ -490,6 +491,8 @@ static struct rpc_clnt *rpc_create_xprt(struct rpc_create_args *args,
+ 		}
+ 	}
+ 
++	rpc_multipath_ops_create_clnt(args, clnt);
++
+ 	clnt->cl_softrtry = 1;
+ 	if (args->flags & RPC_CLNT_CREATE_HARDRTRY)
+ 		clnt->cl_softrtry = 0;
+@@ -869,6 +872,8 @@ void rpc_shutdown_client(struct rpc_clnt *clnt)
+ 			list_empty(&clnt->cl_tasks), 1*HZ);
+ 	}
+ 
++	rpc_multipath_ops_releas_clnt(clnt);
++
+ 	rpc_release_client(clnt);
+ }
+ EXPORT_SYMBOL_GPL(rpc_shutdown_client);
+@@ -981,6 +986,12 @@ void rpc_task_release_transport(struct rpc_task *task)
+ 
+ 	if (xprt) {
+ 		task->tk_xprt = NULL;
++#if IS_ENABLED(CONFIG_ENFS)
++		if (task->tk_client) {
++			rpc_task_release_xprt(task->tk_client, xprt);
++			return;
++		}
++#endif
+ 		xprt_put(xprt);
+ 	}
+ }
+@@ -990,6 +1001,10 @@ void rpc_task_release_client(struct rpc_task *task)
+ {
+ 	struct rpc_clnt *clnt = task->tk_client;
+ 
++#if IS_ENABLED(CONFIG_ENFS)
++	rpc_task_release_transport(task);
++#endif
++
+ 	if (clnt != NULL) {
+ 		/* Remove from client task list */
+ 		spin_lock(&clnt->cl_lock);
+@@ -999,14 +1014,29 @@ void rpc_task_release_client(struct rpc_task *task)
+ 
+ 		rpc_release_client(clnt);
+ 	}
+-	rpc_task_release_transport(task);
++	
++	if (!IS_ENABLED(CONFIG_ENFS))
++		rpc_task_release_transport(task);
++
+ }
+ 
++#if IS_ENABLED(CONFIG_ENFS)
++static struct rpc_xprt *
++rpc_task_get_next_xprt(struct rpc_clnt *clnt)
++{
++	return rpc_task_get_xprt(clnt, xprt_iter_get_next(&clnt->cl_xpi));
++}
++#endif
++
+ static
+ void rpc_task_set_transport(struct rpc_task *task, struct rpc_clnt *clnt)
+ {
+ 	if (!task->tk_xprt)
++#if IS_ENABLED(CONFIG_ENFS)
++		task->tk_xprt = rpc_task_get_next_xprt(clnt);
++#else
+ 		task->tk_xprt = xprt_iter_get_next(&clnt->cl_xpi);
++#endif
+ }
+ 
+ static
+@@ -1597,6 +1627,14 @@ call_reserveresult(struct rpc_task *task)
+ 		return;
+ 	case -EIO:	/* probably a shutdown */
+ 		break;
++#if IS_ENABLED(CONFIG_ENFS)
++	case -ETIMEDOUT:	/* woken up; restart */
++		if (rpc_multipath_ops_task_need_call_start_again(task)) {
++			rpc_task_release_transport(task);
++			task->tk_action = call_start;
++			return;
++		}
++#endif
+ 	default:
+ 		printk(KERN_ERR "%s: unrecognized error %d, exiting\n",
+ 				__func__, status);
+@@ -1962,6 +2000,10 @@ call_transmit(struct rpc_task *task)
+ 		return;
+ 	if (!xprt_prepare_transmit(task))
+ 		return;
++
++	if (rpc_multipath_ops_prepare_transmit(task))
++		return;
++
+ 	task->tk_action = call_transmit_status;
+ 	/* Encode here so that rpcsec_gss can use correct sequence number. */
+ 	if (rpc_task_need_encode(task)) {
+@@ -2277,6 +2319,7 @@ call_timeout(struct rpc_task *task)
+ 
+ retry:
+ 	task->tk_action = call_bind;
++	rpc_multipath_ops_failover_handle(task);
+ 	task->tk_status = 0;
+ }
+ 
+@@ -2961,3 +3004,30 @@ rpc_clnt_swap_deactivate(struct rpc_clnt *clnt)
+ }
+ EXPORT_SYMBOL_GPL(rpc_clnt_swap_deactivate);
+ #endif /* CONFIG_SUNRPC_SWAP */
++
++#if IS_ENABLED(CONFIG_ENFS)
++/* rpc_clnt_test_xprt - Test and add a new transport to a rpc_clnt
++ * @clnt: pointer to struct rpc_clnt
++ * @xprt: pointer struct rpc_xprt
++ * @ops: async operation
++ */
++int
++rpc_clnt_test_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt,
++		   const struct rpc_call_ops *ops, void *data, int flags)
++{
++	struct rpc_cred *cred;
++	struct rpc_task *task;
++
++	cred = authnull_ops.lookup_cred(NULL, NULL, 0);
++	task = rpc_call_null_helper(clnt, xprt, cred,
++				    RPC_TASK_SOFT | RPC_TASK_SOFTCONN | flags,
++				    ops, data);
++	put_rpccred(cred);
++	if (IS_ERR(task))
++		return PTR_ERR(task);
++
++	rpc_put_task(task);
++	return 1;
++}
++EXPORT_SYMBOL_GPL(rpc_clnt_test_xprt);
++#endif
+diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
+index a873c92a4898..2254fea0e863 100644
+--- a/net/sunrpc/sched.c
++++ b/net/sunrpc/sched.c
+@@ -20,7 +20,7 @@
+ #include <linux/mutex.h>
+ #include <linux/freezer.h>
+ 
+-#include <linux/sunrpc/clnt.h>
++#include <linux/sunrpc/sunrpc_enfs_adapter.h>
+ 
+ #include "sunrpc.h"
+ 
+@@ -962,7 +962,12 @@ static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *ta
+ 	/* Initialize workqueue for async tasks */
+ 	task->tk_workqueue = task_setup_data->workqueue;
+ 
++#if IS_ENABLED(CONFIG_ENFS)
++	task->tk_xprt = rpc_task_get_xprt(task_setup_data->rpc_client,
++					  xprt_get(task_setup_data->rpc_xprt));
++#else
+ 	task->tk_xprt = xprt_get(task_setup_data->rpc_xprt);
++#endif
+ 
+ 	if (task->tk_ops->rpc_call_prepare != NULL)
+ 		task->tk_action = rpc_prepare_task;
+diff --git a/net/sunrpc/sunrpc_enfs_adapter.c b/net/sunrpc/sunrpc_enfs_adapter.c
+new file mode 100644
+index 000000000000..c1543545c6de
+--- /dev/null
++++ b/net/sunrpc/sunrpc_enfs_adapter.c
+@@ -0,0 +1,214 @@
++// SPDX-License-Identifier: GPL-2.0
++/* Client-side SUNRPC ENFS adapter header.
++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved.
++ */
++#include <linux/sunrpc/sunrpc_enfs_adapter.h>
++
++struct rpc_multipath_ops __rcu *multipath_ops;
++
++void rpc_init_task_retry_counters(struct rpc_task *task)
++{
++	/* Initialize retry counters */
++	task->tk_garb_retry = 2;
++	task->tk_cred_retry = 2;
++	task->tk_rebind_retry = 2;
++}
++EXPORT_SYMBOL_GPL(rpc_init_task_retry_counters);
++
++struct rpc_xprt *
++rpc_task_get_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt)
++{
++	struct rpc_xprt_switch *xps;
++
++	if (!xprt)
++		return NULL;
++	rcu_read_lock();
++	xps = rcu_dereference(clnt->cl_xpi.xpi_xpswitch);
++	atomic_long_inc(&xps->xps_queuelen);
++	rcu_read_unlock();
++	atomic_long_inc(&xprt->queuelen);
++
++	return xprt;
++}
++
++int rpc_multipath_ops_register(struct rpc_multipath_ops *ops)
++{
++	struct rpc_multipath_ops *old;
++
++	old = cmpxchg((struct rpc_multipath_ops **)&multipath_ops, NULL, ops);
++	if (!old || old == ops)
++		return 0;
++	pr_err("regist rpc_multipath ops %p fail. old %p\n", ops, old);
++	return -EPERM;
++}
++EXPORT_SYMBOL_GPL(rpc_multipath_ops_register);
++
++int rpc_multipath_ops_unregister(struct rpc_multipath_ops *ops)
++{
++	struct rpc_multipath_ops *old;
++
++	old = cmpxchg((struct rpc_multipath_ops **)&multipath_ops, ops, NULL);
++	if (!old || old == ops)
++		return 0;
++	pr_err("regist rpc_multipath ops %p fail. old %p\n", ops, old);
++	return -EPERM;
++}
++EXPORT_SYMBOL_GPL(rpc_multipath_ops_unregister);
++
++struct rpc_multipath_ops *rpc_multipath_ops_get(void)
++{
++	struct rpc_multipath_ops *ops;
++
++	rcu_read_lock();
++	ops = rcu_dereference(multipath_ops);
++	if (!ops) {
++		rcu_read_unlock();
++		return NULL;
++	}
++	if (!try_module_get(ops->owner))
++		ops = NULL;
++	rcu_read_unlock();
++	return ops;
++}
++EXPORT_SYMBOL_GPL(rpc_multipath_ops_get);
++
++void rpc_multipath_ops_put(struct rpc_multipath_ops *ops)
++{
++	if (ops)
++		module_put(ops->owner);
++}
++EXPORT_SYMBOL_GPL(rpc_multipath_ops_put);
++
++void rpc_task_release_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt)
++{
++	struct rpc_xprt_switch *xps;
++
++	atomic_long_dec(&xprt->queuelen);
++	rcu_read_lock();
++	xps = rcu_dereference(clnt->cl_xpi.xpi_xpswitch);
++	atomic_long_dec(&xps->xps_queuelen);
++	rcu_read_unlock();
++
++	xprt_put(xprt);
++}
++
++void rpc_multipath_ops_create_clnt(struct rpc_create_args *args,
++				   struct rpc_clnt *clnt)
++{
++	struct rpc_multipath_ops *mops;
++
++	if (args->multipath_option) {
++		mops = rpc_multipath_ops_get();
++		if (mops && mops->create_clnt)
++			mops->create_clnt(args, clnt);
++		rpc_multipath_ops_put(mops);
++	}
++}
++
++void rpc_multipath_ops_releas_clnt(struct rpc_clnt *clnt)
++{
++	struct rpc_multipath_ops *mops;
++
++	mops = rpc_multipath_ops_get();
++	if (mops && mops->releas_clnt)
++		mops->releas_clnt(clnt);
++
++	rpc_multipath_ops_put(mops);
++}
++
++bool rpc_multipath_ops_create_xprt(struct rpc_xprt *xprt)
++{
++	struct rpc_multipath_ops *mops = NULL;
++
++	mops = rpc_multipath_ops_get();
++	if (mops && mops->create_xprt) {
++		mops->create_xprt(xprt);
++		if (!xprt->multipath_context) {
++			rpc_multipath_ops_put(mops);
++			return true;
++		}
++	}
++	rpc_multipath_ops_put(mops);
++	return false;
++}
++
++void rpc_multipath_ops_destroy_xprt(struct rpc_xprt *xprt)
++{
++	struct rpc_multipath_ops *mops;
++
++	if (xprt->multipath_context) {
++		mops = rpc_multipath_ops_get();
++		if (mops && mops->destroy_xprt)
++			mops->destroy_xprt(xprt);
++		rpc_multipath_ops_put(mops);
++	}
++}
++
++void rpc_multipath_ops_xprt_iostat(struct rpc_task *task)
++{
++	struct rpc_multipath_ops *mops;
++
++	mops = rpc_multipath_ops_get();
++	if (task->tk_client && mops && mops->xprt_iostat)
++		mops->xprt_iostat(task);
++	rpc_multipath_ops_put(mops);
++}
++
++void rpc_multipath_ops_failover_handle(struct rpc_task *task)
++{
++	struct rpc_multipath_ops *mpath_ops = NULL;
++
++	mpath_ops = rpc_multipath_ops_get();
++	if (mpath_ops && mpath_ops->failover_handle)
++		mpath_ops->failover_handle(task);
++	rpc_multipath_ops_put(mpath_ops);
++}
++
++bool rpc_multipath_ops_task_need_call_start_again(struct rpc_task *task)
++{
++	struct rpc_multipath_ops *mpath_ops = NULL;
++	bool ret = false;
++
++	mpath_ops = rpc_multipath_ops_get();
++	if (mpath_ops && mpath_ops->task_need_call_start_again)
++		ret = mpath_ops->task_need_call_start_again(task);
++	rpc_multipath_ops_put(mpath_ops);
++	return ret;
++}
++
++void rpc_multipath_ops_adjust_task_timeout(struct rpc_task *task,
++					   void *condition)
++{
++	struct rpc_multipath_ops *mops = NULL;
++
++	mops = rpc_multipath_ops_get();
++	if (mops && mops->adjust_task_timeout)
++		mops->adjust_task_timeout(task, NULL);
++	rpc_multipath_ops_put(mops);
++}
++
++void rpc_multipath_ops_init_task_req(struct rpc_task *task,
++				     struct rpc_rqst *req)
++{
++	struct rpc_multipath_ops *mops = NULL;
++
++	mops = rpc_multipath_ops_get();
++	if (mops && mops->init_task_req)
++		mops->init_task_req(task, req);
++	rpc_multipath_ops_put(mops);
++}
++
++bool rpc_multipath_ops_prepare_transmit(struct rpc_task *task)
++{
++	struct rpc_multipath_ops *mops = NULL;
++
++	mops = rpc_multipath_ops_get();
++	if (mops && mops->prepare_transmit) {
++		if (!(mops->prepare_transmit(task))) {
++			rpc_multipath_ops_put(mops);
++			return true;
++		}
++	}
++	rpc_multipath_ops_put(mops);
++	return false;
++}
+diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
+index c912bf20faa2..c2b63b3d5217 100644
+--- a/net/sunrpc/xprt.c
++++ b/net/sunrpc/xprt.c
+@@ -48,6 +48,7 @@
+ #include <linux/sunrpc/clnt.h>
+ #include <linux/sunrpc/metrics.h>
+ #include <linux/sunrpc/bc_xprt.h>
++#include <linux/sunrpc/sunrpc_enfs_adapter.h>
+ #include <linux/rcupdate.h>
+ 
+ #include <trace/events/sunrpc.h>
+@@ -259,6 +260,9 @@ int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
+ 	dprintk("RPC: %5u failed to lock transport %p\n",
+ 			task->tk_pid, xprt);
+ 	task->tk_timeout = 0;
++
++	rpc_multipath_ops_adjust_task_timeout(task, NULL);
++
+ 	task->tk_status = -EAGAIN;
+ 	if (req == NULL)
+ 		priority = RPC_PRIORITY_LOW;
+@@ -560,6 +564,9 @@ void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action)
+ 	struct rpc_xprt *xprt = req->rq_xprt;
+ 
+ 	task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0;
++
++	rpc_multipath_ops_adjust_task_timeout(task, NULL);
++
+ 	rpc_sleep_on(&xprt->pending, task, action);
+ }
+ EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space);
+@@ -1347,6 +1354,9 @@ xprt_request_init(struct rpc_task *task)
+ 	req->rq_rcv_buf.buflen = 0;
+ 	req->rq_release_snd_buf = NULL;
+ 	xprt_reset_majortimeo(req);
++
++	rpc_multipath_ops_init_task_req(task, req);
++
+ 	dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid,
+ 			req, ntohl(req->rq_xid));
+ }
+@@ -1427,6 +1437,9 @@ void xprt_release(struct rpc_task *task)
+ 		task->tk_ops->rpc_count_stats(task, task->tk_calldata);
+ 	else if (task->tk_client)
+ 		rpc_count_iostats(task, task->tk_client->cl_metrics);
++
++	rpc_multipath_ops_xprt_iostat(task);
++
+ 	spin_lock(&xprt->recv_lock);
+ 	if (!list_empty(&req->rq_list)) {
+ 		list_del_init(&req->rq_list);
+@@ -1455,6 +1468,7 @@ void xprt_release(struct rpc_task *task)
+ 	else
+ 		xprt_free_bc_request(req);
+ }
++EXPORT_SYMBOL_GPL(xprt_release);
+ 
+ static void xprt_init(struct rpc_xprt *xprt, struct net *net)
+ {
+@@ -1528,6 +1542,10 @@ struct rpc_xprt *xprt_create_transport(struct xprt_create *args)
+ 		return ERR_PTR(-ENOMEM);
+ 	}
+ 
++if (rpc_multipath_ops_create_xprt(xprt)) {
++	xprt_destroy(xprt);
++	return ERR_PTR(-ENOMEM);
++}
+ 	rpc_xprt_debugfs_register(xprt);
+ 
+ 	dprintk("RPC:       created transport %p with %u slots\n", xprt,
+@@ -1547,6 +1565,9 @@ static void xprt_destroy_cb(struct work_struct *work)
+ 	rpc_destroy_wait_queue(&xprt->sending);
+ 	rpc_destroy_wait_queue(&xprt->backlog);
+ 	kfree(xprt->servername);
++
++	rpc_multipath_ops_destroy_xprt(xprt);
++
+ 	/*
+ 	 * Tear down transport state and free the rpc_xprt
+ 	 */
+diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c
+index 6ebaa58b4eff..f6a004ee7a27 100644
+--- a/net/sunrpc/xprtmultipath.c
++++ b/net/sunrpc/xprtmultipath.c
+@@ -18,6 +18,7 @@
+ #include <linux/sunrpc/xprt.h>
+ #include <linux/sunrpc/addr.h>
+ #include <linux/sunrpc/xprtmultipath.h>
++#include <linux/sunrpc/sunrpc_enfs_adapter.h>
+ 
+ typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head,
+ 		const struct rpc_xprt *cur);
+@@ -26,8 +27,8 @@ static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular;
+ static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin;
+ static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall;
+ 
+-static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
+-		struct rpc_xprt *xprt)
++void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
++				 struct rpc_xprt *xprt)
+ {
+ 	if (unlikely(xprt_get(xprt) == NULL))
+ 		return;
+@@ -36,7 +37,9 @@ static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
+ 	if (xps->xps_nxprts == 0)
+ 		xps->xps_net = xprt->xprt_net;
+ 	xps->xps_nxprts++;
++	rpc_xps_nactive_add_one(xps);
+ }
++EXPORT_SYMBOL(xprt_switch_add_xprt_locked);
+ 
+ /**
+  * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch
+@@ -63,6 +66,7 @@ static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps,
+ 	if (unlikely(xprt == NULL))
+ 		return;
+ 	xps->xps_nxprts--;
++	rpc_xps_nactive_sub_one(xps);
+ 	if (xps->xps_nxprts == 0)
+ 		xps->xps_net = NULL;
+ 	smp_wmb();
+@@ -84,7 +88,7 @@ void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps,
+ 	spin_unlock(&xps->xps_lock);
+ 	xprt_put(xprt);
+ }
+-
++EXPORT_SYMBOL(rpc_xprt_switch_remove_xprt);
+ /**
+  * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch
+  * @xprt: pointer to struct rpc_xprt
+@@ -103,6 +107,10 @@ struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt,
+ 		spin_lock_init(&xps->xps_lock);
+ 		kref_init(&xps->xps_kref);
+ 		xps->xps_nxprts = 0;
++#if IS_ENABLED(CONFIG_ENFS)
++		xps->xps_nactive = 0;
++		atomic_long_set(&xps->xps_queuelen, 0);
++#endif
+ 		INIT_LIST_HEAD(&xps->xps_xprt_list);
+ 		xps->xps_iter_ops = &rpc_xprt_iter_singular;
+ 		xprt_switch_add_xprt_locked(xps, xprt);
+@@ -148,6 +156,7 @@ struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps)
+ 		return xps;
+ 	return NULL;
+ }
++EXPORT_SYMBOL(xprt_switch_get);
+ 
+ /**
+  * xprt_switch_put - Release a reference to a rpc_xprt_switch
+@@ -160,6 +169,7 @@ void xprt_switch_put(struct rpc_xprt_switch *xps)
+ 	if (xps != NULL)
+ 		kref_put(&xps->xps_kref, xprt_switch_free);
+ }
++EXPORT_SYMBOL(xprt_switch_put);
+ 
+ /**
+  * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch

0006-add_enfs_compile_option.patch

@@ -0,0 +1,70 @@
+diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig
+index b04256636d4b..ae53510c0627 100644
+--- a/arch/arm64/configs/openeuler_defconfig
++++ b/arch/arm64/configs/openeuler_defconfig
+@@ -5344,6 +5344,7 @@ CONFIG_LOCKD=m
+ CONFIG_LOCKD_V4=y
+ CONFIG_NFS_ACL_SUPPORT=m
+ CONFIG_NFS_COMMON=y
++# CONFIG_ENFS is not set
+ CONFIG_SUNRPC=m
+ CONFIG_SUNRPC_GSS=m
+ CONFIG_SUNRPC_BACKCHANNEL=y
+diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig
+index 59baeb2973af..ccc317f7fdb2 100644
+--- a/arch/x86/configs/openeuler_defconfig
++++ b/arch/x86/configs/openeuler_defconfig
+@@ -6825,6 +6825,7 @@ CONFIG_LOCKD=m
+ CONFIG_LOCKD_V4=y
+ CONFIG_NFS_ACL_SUPPORT=m
+ CONFIG_NFS_COMMON=y
++# CONFIG_ENFS is not set
+ CONFIG_SUNRPC=m
+ CONFIG_SUNRPC_GSS=m
+ CONFIG_SUNRPC_BACKCHANNEL=y
+diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig
+index e55f86713948..872c9b7671b1 100644
+--- a/fs/nfs/Kconfig
++++ b/fs/nfs/Kconfig
+@@ -196,3 +196,14 @@ config NFS_DEBUG
+ 	depends on NFS_FS && SUNRPC_DEBUG
+ 	select CRC32
+ 	default y
++
++config ENFS
++	tristate "NFS client support for ENFS"
++	depends on NFS_FS
++	default n
++	help
++	  This option enables support multipath of the NFS protocol
++	  in the kernel's NFS client.
++	  This feature will improve performance and reliability.
++
++	  If sure, say Y.
+diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile
+index c587e3c4c6a6..19d0ac2ba3b8 100644
+--- a/fs/nfs/Makefile
++++ b/fs/nfs/Makefile
+@@ -12,6 +12,7 @@ nfs-y 			:= client.o dir.o file.o getroot.o inode.o super.o \
+ nfs-$(CONFIG_ROOT_NFS)	+= nfsroot.o
+ nfs-$(CONFIG_SYSCTL)	+= sysctl.o
+ nfs-$(CONFIG_NFS_FSCACHE) += fscache.o fscache-index.o
++nfs-$(CONFIG_ENFS) += enfs_adapter.o
+ 
+ obj-$(CONFIG_NFS_V2) += nfsv2.o
+ nfsv2-y := nfs2super.o proc.o nfs2xdr.o
+@@ -34,3 +35,5 @@ nfsv4-$(CONFIG_NFS_V4_2)	+= nfs42proc.o
+ obj-$(CONFIG_PNFS_FILE_LAYOUT) += filelayout/
+ obj-$(CONFIG_PNFS_BLOCK) += blocklayout/
+ obj-$(CONFIG_PNFS_FLEXFILE_LAYOUT) += flexfilelayout/
++
++obj-$(CONFIG_ENFS) += enfs/
+diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile
+index 090658c3da12..fe4e3b28c5d1 100644
+--- a/net/sunrpc/Makefile
++++ b/net/sunrpc/Makefile
+@@ -19,3 +19,4 @@ sunrpc-$(CONFIG_SUNRPC_DEBUG) += debugfs.o
+ sunrpc-$(CONFIG_SUNRPC_BACKCHANNEL) += backchannel_rqst.o
+ sunrpc-$(CONFIG_PROC_FS) += stats.o
+ sunrpc-$(CONFIG_SYSCTL) += sysctl.o
++sunrpc-$(CONFIG_ENFS) += sunrpc_enfs_adapter.o