From 6223f92d7e99dd9fc58c879c52bb8a445d560129 Mon Sep 17 00:00:00 2001
From: Yanmei-Liu <miaoji.lym@alibaba-inc.com>
Date: Wed, 15 Nov 2023 19:18:26 +0800
Subject: [PATCH 1/7] [+] add new transport parameter max_concurrent_paths

---
 include/xquic/xquic_typedef.h        |  1 +
 src/transport/xqc_cid.c              |  1 +
 src/transport/xqc_conn.c             |  4 ++++
 src/transport/xqc_conn.h             |  1 +
 src/transport/xqc_transport_params.c | 24 ++++++++++++++++++++++++
 src/transport/xqc_transport_params.h | 10 ++++++++--
 6 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/include/xquic/xquic_typedef.h b/include/xquic/xquic_typedef.h
index d429fb1fa..ab5bc666d 100644
--- a/include/xquic/xquic_typedef.h
+++ b/include/xquic/xquic_typedef.h
@@ -83,6 +83,7 @@ typedef struct xqc_cid_s {
     uint8_t             cid_buf[XQC_MAX_CID_LEN];
     uint64_t            cid_seq_num;
     uint8_t             sr_token[XQC_STATELESS_RESET_TOKENLEN];
+    uint64_t            path_id;    /* preallocate for multi-path */
 } xqc_cid_t;
 
 typedef enum xqc_log_level_s {
diff --git a/src/transport/xqc_cid.c b/src/transport/xqc_cid.c
index 68a9d3ff0..9739ba53e 100644
--- a/src/transport/xqc_cid.c
+++ b/src/transport/xqc_cid.c
@@ -64,6 +64,7 @@ xqc_cid_copy(xqc_cid_t *dst, xqc_cid_t *src)
     xqc_memcpy(dst->cid_buf, src->cid_buf, dst->cid_len);
     dst->cid_seq_num = src->cid_seq_num;
     xqc_memcpy(dst->sr_token, src->sr_token, XQC_STATELESS_RESET_TOKENLEN);
+    dst->path_id = src->path_id;
 }
 
 void
diff --git a/src/transport/xqc_conn.c b/src/transport/xqc_conn.c
index c83a66033..b822a33ab 100644
--- a/src/transport/xqc_conn.c
+++ b/src/transport/xqc_conn.c
@@ -316,6 +316,7 @@ xqc_conn_set_default_settings(xqc_trans_settings_t *settings)
     settings->ack_delay_exponent         = XQC_DEFAULT_ACK_DELAY_EXPONENT;
     settings->max_udp_payload_size       = XQC_DEFAULT_MAX_UDP_PAYLOAD_SIZE;
     settings->active_connection_id_limit = XQC_DEFAULT_ACTIVE_CONNECTION_ID_LIMIT;
+    settings->max_concurrent_paths       = XQC_DEFAULT_MAX_CONCURRENT_PATHS;
 }
 
 static inline void
@@ -4574,6 +4575,7 @@ xqc_conn_set_remote_transport_params(xqc_connection_t *conn,
 
     settings->enable_multipath = params->enable_multipath;
     settings->multipath_version = params->multipath_version;
+    settings->max_concurrent_paths = params->max_concurrent_paths;
     settings->max_datagram_frame_size = params->max_datagram_frame_size;
 
     return XQC_OK;
@@ -4611,6 +4613,7 @@ xqc_conn_get_local_transport_params(xqc_connection_t *conn, xqc_transport_params
     params->no_crypto = settings->no_crypto;
     params->enable_multipath = settings->enable_multipath;
     params->multipath_version = settings->multipath_version;
+    params->max_concurrent_paths = settings->max_concurrent_paths;
     params->max_datagram_frame_size = settings->max_datagram_frame_size;
 
     /* set other transport parameters */
@@ -4911,6 +4914,7 @@ xqc_settings_copy_from_transport_params(xqc_trans_settings_t *dest,
     dest->active_connection_id_limit = src->active_connection_id_limit;
 
     dest->enable_multipath = src->enable_multipath;
+    dest->max_concurrent_paths = src->max_concurrent_paths;
     dest->max_datagram_frame_size = src->max_datagram_frame_size;
 }
 
diff --git a/src/transport/xqc_conn.h b/src/transport/xqc_conn.h
index 2700a8d6d..28e0fa6ce 100644
--- a/src/transport/xqc_conn.h
+++ b/src/transport/xqc_conn.h
@@ -211,6 +211,7 @@ typedef struct {
     uint64_t                enable_multipath;
     xqc_multipath_version_t multipath_version;
     uint16_t                max_datagram_frame_size;
+    uint64_t                max_concurrent_paths;
 } xqc_trans_settings_t;
 
 
diff --git a/src/transport/xqc_transport_params.c b/src/transport/xqc_transport_params.c
index 944ee5e63..de7063a8f 100644
--- a/src/transport/xqc_transport_params.c
+++ b/src/transport/xqc_transport_params.c
@@ -172,6 +172,12 @@ xqc_transport_params_calc_length(const xqc_transport_params_t *params,
                 xqc_put_varint_len(xqc_put_varint_len(params->enable_multipath)) +
                 xqc_put_varint_len(params->enable_multipath);
         }
+
+        if (params->max_concurrent_paths != XQC_DEFAULT_MAX_CONCURRENT_PATHS) {
+            len += xqc_put_varint_len(XQC_TRANSPORT_PARAM_MAX_CONCURRENT_PATHS) +
+                   xqc_put_varint_len(xqc_put_varint_len(params->max_concurrent_paths)) +
+                   xqc_put_varint_len(params->max_concurrent_paths);
+        }
     }
 
     if (params->max_datagram_frame_size) {
@@ -353,6 +359,11 @@ xqc_encode_transport_params(const xqc_transport_params_t *params,
         } else {
             p = xqc_put_varint_param(p, XQC_TRANSPORT_PARAM_ENABLE_MULTIPATH_04, params->enable_multipath);
         }
+
+        if (params->max_concurrent_paths != XQC_DEFAULT_MAX_CONCURRENT_PATHS) {
+            p = xqc_put_varint_param(p, XQC_TRANSPORT_PARAM_MAX_CONCURRENT_PATHS,
+                                     params->max_concurrent_paths);
+        }
     }
 
     if ((size_t)(p - out) != len) {
@@ -615,6 +626,13 @@ xqc_decode_enable_multipath(xqc_transport_params_t *params, xqc_transport_params
     return XQC_OK;
 }
 
+static xqc_int_t
+xqc_decode_max_concurrent_paths(xqc_transport_params_t *params, xqc_transport_params_type_t exttype,
+    const uint8_t *p, const uint8_t *end, uint64_t param_type, uint64_t param_len)
+{
+    XQC_DECODE_VINT_VALUE(&params->max_concurrent_paths, p, end);
+}
+
 static xqc_int_t
 xqc_decode_max_datagram_frame_size(xqc_transport_params_t *params, xqc_transport_params_type_t exttype,
     const uint8_t *p, const uint8_t *end, uint64_t param_type, uint64_t param_len)
@@ -649,6 +667,7 @@ xqc_trans_param_decode_func xqc_trans_param_decode_func_list[] = {
     xqc_decode_no_crypto,
     xqc_decode_enable_multipath,
     xqc_decode_max_datagram_frame_size,
+    xqc_decode_max_concurrent_paths,
 };
 
 
@@ -688,6 +707,9 @@ xqc_trans_param_get_index(uint64_t param_type)
     case XQC_TRANSPORT_PARAM_MAX_DATAGRAM_FRAME_SIZE:
         return XQC_TRANSPORT_PARAM_PROTOCOL_MAX + 2;
 
+    case XQC_TRANSPORT_PARAM_MAX_CONCURRENT_PATHS:
+        return XQC_TRANSPORT_PARAM_PROTOCOL_MAX + 3;
+
     default:
         break;
     }
@@ -780,6 +802,7 @@ xqc_decode_transport_params(xqc_transport_params_t *params,
 
     params->enable_multipath = 0;
     params->multipath_version = XQC_ERR_MULTIPATH_VERSION;
+    params->max_concurrent_paths = XQC_DEFAULT_MAX_CONCURRENT_PATHS;
 
     while (p < end) {
         ret = xqc_decode_one_transport_param(params, exttype, &p, end);
@@ -910,4 +933,5 @@ xqc_init_transport_params(xqc_transport_params_t *params)
     params->ack_delay_exponent = XQC_DEFAULT_ACK_DELAY_EXPONENT;
     params->max_udp_payload_size = XQC_DEFAULT_MAX_UDP_PAYLOAD_SIZE;
     params->active_connection_id_limit = XQC_DEFAULT_ACTIVE_CONNECTION_ID_LIMIT;
+    params->max_concurrent_paths = XQC_DEFAULT_MAX_CONCURRENT_PATHS;
 }
diff --git a/src/transport/xqc_transport_params.h b/src/transport/xqc_transport_params.h
index 2abbf6897..9daa62235 100644
--- a/src/transport/xqc_transport_params.h
+++ b/src/transport/xqc_transport_params.h
@@ -23,7 +23,8 @@
 /* max buffer length of encoded transport parameter */
 #define XQC_MAX_TRANSPORT_PARAM_BUF_LEN         512
 
-
+/* default value for max_concurrent_paths */
+#define XQC_DEFAULT_MAX_CONCURRENT_PATHS        4
 
 /**
  * @brief transport parameter type
@@ -74,6 +75,9 @@ typedef enum {
     XQC_TRANSPORT_PARAM_ENABLE_MULTIPATH_05                 = 0x0f739bbc1b666d05,
     XQC_TRANSPORT_PARAM_ENABLE_MULTIPATH_06                 = 0x0f739bbc1b666d06,
 
+    /* multipath max concurrent paths */
+    XQC_TRANSPORT_PARAM_MAX_CONCURRENT_PATHS                = 0x0f739bbc1b666df1,
+
     /* upper limit of params defined by xquic */
     XQC_TRANSPORT_PARAM_UNKNOWN,
 } xqc_transport_param_id_t;
@@ -146,7 +150,9 @@ typedef struct {
     uint64_t                enable_multipath;
 
 
-    xqc_multipath_version_t   multipath_version;
+    xqc_multipath_version_t multipath_version;
+
+    uint64_t                max_concurrent_paths;
 
 } xqc_transport_params_t;
 

From c085d0040a867326555df6b4f692c12a2c971e78 Mon Sep 17 00:00:00 2001
From: Yanmei-Liu <miaoji.lym@alibaba-inc.com>
Date: Wed, 15 Nov 2023 19:43:22 +0800
Subject: [PATCH 2/7] [+] add option for demo client

---
 demo/demo_client.c    | 15 +++++++++++++--
 include/xquic/xquic.h |  1 +
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/demo/demo_client.c b/demo/demo_client.c
index e00370389..2f91021dc 100644
--- a/demo/demo_client.c
+++ b/demo/demo_client.c
@@ -171,6 +171,8 @@ typedef struct xqc_demo_cli_quic_config_s {
 
     uint8_t mp_version;
 
+    uint64_t max_concurrent_paths;
+
     /* support interop test */
     int is_interop_mode;
 
@@ -1629,6 +1631,7 @@ xqc_demo_cli_init_conneciton_settings(xqc_conn_settings_t* settings,
     settings->standby_path_probe_timeout = 1000;
     settings->multipath_version = args->quic_cfg.mp_version;
     settings->mp_ping_on = 1;
+    settings->max_concurrent_paths = args->quic_cfg.max_concurrent_paths;
     settings->is_interop_mode = args->quic_cfg.is_interop_mode;
     if (args->req_cfg.throttled_req != -1) {
         settings->enable_stream_rate_limit = 1;
@@ -1659,6 +1662,7 @@ xqc_demo_cli_init_args(xqc_demo_cli_client_args_t *args)
     args->quic_cfg.keyupdate_pkt_threshold = UINT64_MAX;
     /* default 04 */
     args->quic_cfg.mp_version = XQC_MULTIPATH_04;
+    args->quic_cfg.max_concurrent_paths = UINT64_MAX;
 
     args->req_cfg.throttled_req = -1;
 
@@ -1774,6 +1778,7 @@ xqc_demo_cli_usage(int argc, char *argv[])
         "   -R    Reinjection (1,2,4) \n"
         "   -V    Multipath Version (4,5,6)\n"
         "   -B    Set initial path standby after recvd first application data, and set initial path available after X ms\n"
+        "   -f    Max concurrent paths\n"
         "   -I    Idle interval between requests (ms)\n"
         "   -n    Throttling the {1,2,...}xn-th requests\n"
         "   -e    NAT rebinding on path 0\n"
@@ -1785,7 +1790,7 @@ void
 xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args)
 {
     int ch = 0;
-    while ((ch = getopt(argc, argv, "a:p:c:Ct:S:0m:A:D:l:L:k:K:U:u:dMoi:w:Ps:bZ:NQT:R:V:B:I:n:eE")) != -1) {
+    while ((ch = getopt(argc, argv, "a:p:c:Ct:S:0m:A:D:l:L:k:K:U:u:dMoi:w:Ps:bZ:NQT:R:V:B:I:n:eEf:")) != -1) {
         switch (ch) {
         /* server ip */
         case 'a':
@@ -2015,7 +2020,13 @@ xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args
         case 'E':
             printf("option rebinding path1 after 3s\n");
             args->net_cfg.rebind_p1 = 1;
-            break;     
+            break;
+
+        /* key update packet threshold */
+        case 'f':
+            printf("max concurrent paths: %s\n", optarg);
+            args->quic_cfg.max_concurrent_paths = atoi(optarg);
+            break;
 
         default:
             printf("other option :%c\n", ch);
diff --git a/include/xquic/xquic.h b/include/xquic/xquic.h
index 6dafb870e..48c303186 100644
--- a/include/xquic/xquic.h
+++ b/include/xquic/xquic.h
@@ -1171,6 +1171,7 @@ typedef struct xqc_conn_settings_s {
      */
     uint64_t                    enable_multipath;
     xqc_multipath_version_t     multipath_version;
+    uint64_t                    max_concurrent_paths;
     uint64_t                    least_available_cid_count;
 
     /*

From bf4d09110a98c7d348465195b9db03748cacdea7 Mon Sep 17 00:00:00 2001
From: Yanmei-Liu <miaoji.lym@alibaba-inc.com>
Date: Wed, 15 Nov 2023 20:35:53 +0800
Subject: [PATCH 3/7] [+] get min(local, remote) for max_concurrent_paths

---
 demo/demo_client.c       | 2 +-
 src/transport/xqc_conn.c | 3 +++
 src/transport/xqc_conn.h | 4 +++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/demo/demo_client.c b/demo/demo_client.c
index 2f91021dc..74c113ba1 100644
--- a/demo/demo_client.c
+++ b/demo/demo_client.c
@@ -1662,7 +1662,7 @@ xqc_demo_cli_init_args(xqc_demo_cli_client_args_t *args)
     args->quic_cfg.keyupdate_pkt_threshold = UINT64_MAX;
     /* default 04 */
     args->quic_cfg.mp_version = XQC_MULTIPATH_04;
-    args->quic_cfg.max_concurrent_paths = UINT64_MAX;
+    args->quic_cfg.max_concurrent_paths = 4;
 
     args->req_cfg.throttled_req = -1;
 
diff --git a/src/transport/xqc_conn.c b/src/transport/xqc_conn.c
index b822a33ab..8a0002a7e 100644
--- a/src/transport/xqc_conn.c
+++ b/src/transport/xqc_conn.c
@@ -372,6 +372,7 @@ xqc_conn_init_trans_settings(xqc_connection_t *conn)
     ls->enable_multipath = conn->conn_settings.enable_multipath;
     
     ls->multipath_version = conn->conn_settings.multipath_version;
+    ls->max_concurrent_paths = conn->conn_settings.max_concurrent_paths;
 
     ls->max_datagram_frame_size = conn->conn_settings.max_datagram_frame_size;
     ls->disable_active_migration = ls->enable_multipath ? 0 : 1;
@@ -4737,6 +4738,8 @@ xqc_conn_tls_transport_params_cb(const uint8_t *tp, size_t len, void *user_data)
     xqc_log(conn->log, XQC_LOG_DEBUG, "|1RTT_transport_params|max_datagram_frame_size:%ud|",
             conn->remote_settings.max_datagram_frame_size);
 
+    conn->max_concurrent_paths = xqc_min(conn->local_settings.max_concurrent_paths,
+                                         conn->remote_settings.max_concurrent_paths);
 
     /* save no crypto flag */
     if (params.no_crypto == 1) {
diff --git a/src/transport/xqc_conn.h b/src/transport/xqc_conn.h
index 28e0fa6ce..6a2bbd853 100644
--- a/src/transport/xqc_conn.h
+++ b/src/transport/xqc_conn.h
@@ -362,7 +362,9 @@ struct xqc_connection_s {
     uint32_t                        create_path_count;
     uint32_t                        validated_path_count;
     uint32_t                        active_path_count;
-    
+
+    uint64_t                        max_concurrent_paths;
+
     const
     xqc_scheduler_callback_t       *scheduler_callback;
     void                           *scheduler;

From b354ac7ac2e0f6357f6e8e692976d58168948745 Mon Sep 17 00:00:00 2001
From: Wu Zhao <zhaowu.zw@alibaba-inc.com>
Date: Thu, 16 Nov 2023 10:49:27 +0800
Subject: [PATCH 4/7] [!] bugfix on multipath, add scheduler according to path
 quality and path status (#370)

* [!] fix compile error on macos

* [!] downgrade the required version of libevent

* [!] fix compability with elder versions

* [=] delete duplicated macro

* [=] code format

* [!] fix sending duplicated ACK when no pkts received on Standby paths.
[+] reinject frames which will affect stream delivery.
[~] optimize mpquic scheduler according to path qulity and path status.
[!] fix sending RESET_STREAM  and STOP_SENDING frames in Initial and Handshake packets.
[!] fix delivery priority of HANDSHAKE_DONE frame blocked by server's STREAM frames responding to 0RTT requests.
[!] fix CID rotation when multipath is enabled.
---
 demo/demo_client.c                            |   2 +-
 include/xquic/xqc_configure.h                 |  10 +-
 include/xquic/xquic.h                         |  23 ++-
 scripts/case_test.sh                          |   7 +-
 scripts/xquic.lds                             |   4 +-
 src/common/utils/ringarray/xqc_ring_array.c   |   8 +
 src/common/utils/ringarray/xqc_ring_array.h   |   2 +
 src/http3/xqc_h3_request.c                    |  31 +---
 src/http3/xqc_h3_stream.c                     |   2 +
 src/http3/xqc_h3_stream.h                     |   2 +
 .../reinjection_control/xqc_reinj_deadline.c  |  73 +++++++--
 .../reinjection_control/xqc_reinj_default.c   |  10 +-
 .../reinjection_control/xqc_reinj_dgram.c     |   2 +-
 .../scheduler/xqc_scheduler_backup.c          | 123 +++++++++------
 .../scheduler/xqc_scheduler_minrtt.c          |  88 +++++++----
 src/transport/xqc_conn.c                      | 145 ++++++++++++------
 src/transport/xqc_conn.h                      |  31 ++--
 src/transport/xqc_engine.c                    |  17 +-
 src/transport/xqc_multipath.c                 |  77 +++++++++-
 src/transport/xqc_multipath.h                 |  32 +++-
 src/transport/xqc_packet_out.c                |  86 ++++++++---
 src/transport/xqc_packet_parser.c             |   5 +
 src/transport/xqc_recv_record.c               |  13 +-
 src/transport/xqc_recv_record.h               |   2 +
 src/transport/xqc_reinjection.c               |   5 +-
 src/transport/xqc_send_ctl.c                  |  31 ++++
 src/transport/xqc_send_ctl.h                  |   8 +
 src/transport/xqc_timer.c                     |   3 +-
 28 files changed, 594 insertions(+), 248 deletions(-)

diff --git a/demo/demo_client.c b/demo/demo_client.c
index e00370389..1738a5692 100644
--- a/demo/demo_client.c
+++ b/demo/demo_client.c
@@ -1926,7 +1926,7 @@ xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args
             printf("option multipath on\n");
             args->net_cfg.multipath = 1;
             break;
-        
+
          case 'o':
             printf("set interop mode\n");
             args->quic_cfg.is_interop_mode = 1;
diff --git a/include/xquic/xqc_configure.h b/include/xquic/xqc_configure.h
index 5610c8b4c..85b40a454 100644
--- a/include/xquic/xqc_configure.h
+++ b/include/xquic/xqc_configure.h
@@ -2,9 +2,9 @@
 #define xquic_VERSION_MAJOR 0
 #define xquic_VERSION_MINOR 1
 
-#define XQC_ENABLE_BBR2
-#define XQC_ENABLE_RENO
-#define XQC_ENABLE_COPA
-#define XQC_ENABLE_UNLIMITED
-#define XQC_ENABLE_MP_INTEROP
+/* #undef XQC_ENABLE_BBR2 */
+/* #undef XQC_ENABLE_RENO */
+/* #undef XQC_ENABLE_COPA */
+/* #undef XQC_ENABLE_UNLIMITED */
+/* #undef XQC_ENABLE_MP_INTEROP */
 /* #undef XQC_NO_PID_PACKET_PROCESS */
diff --git a/include/xquic/xquic.h b/include/xquic/xquic.h
index 6dafb870e..c8581d78b 100644
--- a/include/xquic/xquic.h
+++ b/include/xquic/xquic.h
@@ -838,20 +838,12 @@ typedef struct xqc_cc_params_s {
 } xqc_cc_params_t;
 
 typedef struct xqc_scheduler_params_u {
-    uint8_t customize_on;
-    union {
-        struct {
-            double lambda_max;
-            double lambda_min;
-            double lambda_init;
-            double lambda_inc;
-            double lambda_dec;
-        } blest_params;
-
-        struct {
-            double beta;
-        } ecf_params;
-    };
+    uint64_t    rtt_us_thr_high;
+    uint64_t    rtt_us_thr_low;
+    uint64_t    bw_Bps_thr;
+    double      loss_percent_thr_high;
+    double      loss_percent_thr_low;
+    uint32_t    pto_cnt_thr;
 } xqc_scheduler_params_t;
 
 typedef struct xqc_congestion_control_callback_s {
@@ -1330,6 +1322,9 @@ typedef struct xqc_conn_stats_s {
      */
     int                 mp_state;
 
+    int                 total_rebind_count;
+    int                 total_rebind_valid;
+
     xqc_path_metrics_t  paths_info[XQC_MAX_PATHS_COUNT];
     char                conn_info[XQC_CONN_INFO_LEN];
 
diff --git a/scripts/case_test.sh b/scripts/case_test.sh
index 9782ab3f6..75479bd09 100755
--- a/scripts/case_test.sh
+++ b/scripts/case_test.sh
@@ -3460,15 +3460,12 @@ clear_log
 echo -e "h3_ext_finish_bytestream_during_transmission...\c"
 ./test_client -l d -T 2 -s 102400 -U 1 -Q 65535 -E -x 304 > stdlog
 cli_res1=`grep ">>>>>>>> pass:1" stdlog | wc -l`
-cli_res2=`grep "\[dgram\]|echo_check|same_content:yes|" stdlog`
-cli_res3=(`grep "\[h3-dgram\]|recv_dgram_bytes:" stdlog | egrep -o ':[0-9]+' | egrep -o '[0-9]+'`)
 cli_res4=(`grep "\[bytestream\]|bytes_sent:" stdlog | egrep -o ':[0-9]+' | egrep -o '[0-9]+'`)
 cli_res5=`grep "\[bytestream\]|same_content:yes|" stdlog | wc -l`
 cli_res6=`grep "send pure fin" clog`
 errlog=`grep_err_log | grep -v "send data after fin sent"`
-if [ "$cli_res1" == "1" ] && [ -n "$cli_res2" ] \
-    && [ ${cli_res3[0]} -ge 102400 ] && [ ${cli_res3[1]} -ge 102400 ] \
-    && [ ${cli_res4[0]} -ge 102400 ] && [ ${cli_res4[1]} -ge 102400 ] \
+if [ "$cli_res1" == "1" ] \
+    && [ ${cli_res4[0]} -eq 102400 ] && [ ${cli_res4[1]} -eq 102400 ] \
     && [ "$cli_res5" == "1" ] && [ -n "$cli_res6" ] && [ -z "$errlog" ]; then
     echo ">>>>>>>> pass:1"
     case_print_result "h3_ext_finish_bytestream_during_transmission" "pass"
diff --git a/scripts/xquic.lds b/scripts/xquic.lds
index a971851c2..dbf3badd6 100644
--- a/scripts/xquic.lds
+++ b/scripts/xquic.lds
@@ -116,8 +116,8 @@ XQUIC_VERS_1.0 {
         xqc_conn_get_conn_settings_template;
         xqc_conn_get_lastest_rtt;
         xqc_log_enable;
-	xqc_h3_request_update_settings;
-  	xqc_stream_update_settings;
+        xqc_h3_request_update_settings;
+        xqc_stream_update_settings;
     local:
         *;
 };
diff --git a/src/common/utils/ringarray/xqc_ring_array.c b/src/common/utils/ringarray/xqc_ring_array.c
index 45d8bc037..90d8cba88 100644
--- a/src/common/utils/ringarray/xqc_ring_array.c
+++ b/src/common/utils/ringarray/xqc_ring_array.c
@@ -239,3 +239,11 @@ xqc_rarray_resize(xqc_rarray_t *ra, uint64_t cap)
 
     return XQC_OK;
 }
+
+void 
+xqc_rarray_reinit(xqc_rarray_t *ra)
+{
+    xqc_memzero(ra->buf, ra->cap * ra->esize);
+    ra->count = 0;
+    ra->offset = 0;
+}
\ No newline at end of file
diff --git a/src/common/utils/ringarray/xqc_ring_array.h b/src/common/utils/ringarray/xqc_ring_array.h
index e495cf5ce..41ad2d683 100644
--- a/src/common/utils/ringarray/xqc_ring_array.h
+++ b/src/common/utils/ringarray/xqc_ring_array.h
@@ -84,4 +84,6 @@ xqc_int_t xqc_rarray_pop_from(xqc_rarray_t *ra, uint64_t idx);
  */
 xqc_int_t xqc_rarray_resize(xqc_rarray_t *ra, uint64_t cap);
 
+void xqc_rarray_reinit(xqc_rarray_t *ra);
+
 #endif
diff --git a/src/http3/xqc_h3_request.c b/src/http3/xqc_h3_request.c
index 7c6e74e4f..9e8c3ce15 100644
--- a/src/http3/xqc_h3_request.c
+++ b/src/http3/xqc_h3_request.c
@@ -177,7 +177,7 @@ xqc_stream_info_print(xqc_h3_stream_t *h3_stream, xqc_request_stats_t *stats)
     size_t cursor = 0, ret = 0;
     int i;
     int flag = 0;
-    uint32_t mp_settings = 0;
+    char mp_settings[XQC_MP_SETTINGS_STR_LEN] = {0};
 
     if (h3c->conn->handshake_complete_time > 0) {
         flag = 1;
@@ -187,32 +187,11 @@ xqc_stream_info_print(xqc_h3_stream_t *h3_stream, xqc_request_stats_t *stats)
         flag |= 1 << 1;
     }
 
-    if (h3c->conn->enable_multipath) {
-        mp_settings |= 1;
-    }
-
-    if (h3c->conn->local_settings.enable_multipath) {
-        mp_settings |= (1 << 1);
-    }
-
-    if (h3c->conn->remote_settings.enable_multipath) {
-        mp_settings |= (1 << 2);
-    }
-
-    if (h3c->conn->conn_settings.multipath_version == XQC_MULTIPATH_05) {
-        mp_settings |= (1 << 3);
-    }
-
-    if (h3c->conn->local_settings.multipath_version == XQC_MULTIPATH_05) {
-        mp_settings |= (1 << 4);
-    }
-
-    if (h3c->conn->remote_settings.multipath_version == XQC_MULTIPATH_05) {
-        mp_settings |= (1 << 5);
-    }
+    xqc_conn_encode_mp_settings(h3c->conn, mp_settings, XQC_MP_SETTINGS_STR_LEN);
 
-    ret = snprintf(buff, buff_size, "(%d,%"PRIu64",%d)#", 
-                   flag, h3_stream->recv_rate_limit, mp_settings);
+    ret = snprintf(buff, buff_size, "(%d,%"PRIu64",%s,%"PRIu64",%"PRIu64")#", 
+                   flag, h3_stream->recv_rate_limit, mp_settings,
+                   h3_stream->send_offset, h3_stream->recv_offset);
 
     cursor += ret;
 
diff --git a/src/http3/xqc_h3_stream.c b/src/http3/xqc_h3_stream.c
index 9c960b062..7e19d1f46 100644
--- a/src/http3/xqc_h3_stream.c
+++ b/src/http3/xqc_h3_stream.c
@@ -1908,6 +1908,8 @@ xqc_h3_stream_close_notify(xqc_stream_t *stream, void *user_data)
         h3s->h3r->stream_fin_send_time = h3s->stream->stream_stats.local_fin_snd_time;
         h3s->h3r->stream_fin_ack_time = h3s->stream->stream_stats.first_fin_ack_time;
         h3s->h3r->stream_close_msg = h3s->stream->stream_close_msg;
+        h3s->send_offset = h3s->stream->stream_send_offset;
+        h3s->recv_offset = h3s->stream->stream_data_in.merged_offset_end;
     }
 
     if (h3s->h3_ext_bs && h3s->type == XQC_H3_STREAM_TYPE_BYTESTEAM) {
diff --git a/src/http3/xqc_h3_stream.h b/src/http3/xqc_h3_stream.h
index b1a151149..dc4526446 100644
--- a/src/http3/xqc_h3_stream.h
+++ b/src/http3/xqc_h3_stream.h
@@ -148,6 +148,8 @@ typedef struct xqc_h3_stream_s {
     uint32_t                        ref_cnt;
 
     uint64_t                        recv_rate_limit;
+    uint64_t                        send_offset;
+    uint64_t                        recv_offset;
 
 } xqc_h3_stream_t;
 
diff --git a/src/transport/reinjection_control/xqc_reinj_deadline.c b/src/transport/reinjection_control/xqc_reinj_deadline.c
index 60d29a716..420890088 100644
--- a/src/transport/reinjection_control/xqc_reinj_deadline.c
+++ b/src/transport/reinjection_control/xqc_reinj_deadline.c
@@ -42,10 +42,37 @@ xqc_deadline_reinj_ctl_init(void *reinj_ctl, xqc_connection_t *conn)
     rctl->conn = conn;
 }
 
+static inline xqc_bool_t
+xqc_deadline_reinj_check_packet(xqc_packet_out_t *po)
+{
+    if (((po->po_frame_types & XQC_FRAME_BIT_STREAM) 
+         || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA)
+         || (po->po_frame_types & XQC_FRAME_BIT_RESET_STREAM)
+         || (po->po_frame_types & XQC_FRAME_BIT_STOP_SENDING)
+         || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAMS)
+         || (po->po_frame_types & XQC_FRAME_BIT_MAX_DATA)
+         || (po->po_frame_types & XQC_FRAME_BIT_DATA_BLOCKED)
+         || (po->po_frame_types & XQC_FRAME_BIT_STREAM_DATA_BLOCKED)
+         || (po->po_frame_types & XQC_FRAME_BIT_STREAMS_BLOCKED)
+         || (po->po_frame_types & XQC_FRAME_BIT_CONNECTION_CLOSE))
+        && !(po->po_flag & XQC_POF_NOT_REINJECT)
+        && !(XQC_MP_PKT_REINJECTED(po))
+        && (po->po_flag & XQC_POF_IN_FLIGHT)) 
+    {   
+        return XQC_TRUE;
+    }
+
+    return XQC_FALSE;
+}
+
 static xqc_bool_t
 xqc_deadline_reinj_can_reinject_before_sched(xqc_deadline_reinj_ctl_t *rctl, 
     xqc_packet_out_t *po)
 {
+    if (!xqc_deadline_reinj_check_packet(po)) {
+        return XQC_FALSE;
+    }
+
     xqc_connection_t *conn = rctl->conn;
     xqc_usec_t now = xqc_monotonic_timestamp();
     xqc_usec_t min_srtt = xqc_conn_get_min_srtt(conn, 0);
@@ -56,22 +83,47 @@ xqc_deadline_reinj_can_reinject_before_sched(xqc_deadline_reinj_ctl_t *rctl,
     uint64_t lower_bound = conn->conn_settings.reinj_deadline_lower_bound;
     double   deadline    = xqc_max(xqc_min(flexible, (double)hard), (double)lower_bound);
 
-    xqc_log(conn->log, XQC_LOG_DEBUG, "|deadline:%f|factor:%.4f|min_srtt:%ui|flexible:%f|hard:%ui|lower_bound:%ui|now:%ui|sent_time:%ui|frame:%s|",
-                                      deadline, factor, min_srtt, flexible, hard, lower_bound, now, po->po_sent_time, xqc_frame_type_2_str(po->po_frame_types));
+    xqc_log(conn->log, XQC_LOG_DEBUG, 
+            "|deadline:%f|factor:%.4f|min_srtt:%ui|flexible:%f|hard:%ui|"
+            "lower_bound:%ui|now:%ui|sent_time:%ui|frame:%s|",
+            deadline, factor, min_srtt, flexible, hard, 
+            lower_bound, now, po->po_sent_time, 
+            xqc_frame_type_2_str(po->po_frame_types));
 
-    if (((po->po_frame_types & XQC_FRAME_BIT_STREAM) 
-         || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA))
-        && !(po->po_flag & XQC_POF_NOT_REINJECT)
-        && !(XQC_MP_PKT_REINJECTED(po))
-        && (po->po_flag & XQC_POF_IN_FLIGHT)
-        && ((double)(now - po->po_sent_time) >= deadline)) 
-    {   
+    if ((double)(now - po->po_sent_time) >= deadline) {   
         return XQC_TRUE;
     }
 
     return XQC_FALSE;
 }
 
+
+static xqc_bool_t
+xqc_deadline_reinj_can_reinject_after_send(xqc_deadline_reinj_ctl_t *rctl, 
+    xqc_packet_out_t *po)
+{
+    if (!xqc_deadline_reinj_check_packet(po)) {
+        return XQC_FALSE;
+    }
+
+    xqc_connection_t *conn = rctl->conn;
+    xqc_path_ctx_t *path = NULL;
+    xqc_path_perf_class_t path_class;
+
+    path = xqc_conn_find_path_by_path_id(conn, po->po_path_id);
+    
+    if (path) {
+        path_class = xqc_path_get_perf_class(path);
+        if (path_class == XQC_PATH_CLASS_STANDBY_LOW 
+            || path_class == XQC_PATH_CLASS_AVAILABLE_LOW)
+        {
+            return XQC_TRUE;
+        }
+    }
+    
+    return XQC_FALSE;
+}
+
 static xqc_bool_t
 xqc_deadline_reinj_can_reinject(void *ctl, 
     xqc_packet_out_t *po, xqc_reinjection_mode_t mode)
@@ -83,6 +135,9 @@ xqc_deadline_reinj_can_reinject(void *ctl,
     case XQC_REINJ_UNACK_BEFORE_SCHED:
         can_reinject = xqc_deadline_reinj_can_reinject_before_sched(rctl, po);
         break;
+    case XQC_REINJ_UNACK_AFTER_SEND:
+        can_reinject = xqc_deadline_reinj_can_reinject_after_send(rctl, po);
+        break;
     default:
         can_reinject = XQC_FALSE;
         break;
diff --git a/src/transport/reinjection_control/xqc_reinj_default.c b/src/transport/reinjection_control/xqc_reinj_default.c
index 265ada8b1..ba2d92a48 100644
--- a/src/transport/reinjection_control/xqc_reinj_default.c
+++ b/src/transport/reinjection_control/xqc_reinj_default.c
@@ -50,7 +50,15 @@ xqc_default_reinj_can_reinject_after_sched(xqc_default_reinj_ctl_t *rctl,
 
     if (xqc_list_empty(&conn->conn_send_queue->sndq_send_packets)
         && ((po->po_frame_types & XQC_FRAME_BIT_STREAM) 
-            || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA))
+            || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA)
+            || (po->po_frame_types & XQC_FRAME_BIT_RESET_STREAM)
+            || (po->po_frame_types & XQC_FRAME_BIT_STOP_SENDING)
+            || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAMS)
+            || (po->po_frame_types & XQC_FRAME_BIT_MAX_DATA)
+            || (po->po_frame_types & XQC_FRAME_BIT_DATA_BLOCKED)
+            || (po->po_frame_types & XQC_FRAME_BIT_STREAM_DATA_BLOCKED)
+            || (po->po_frame_types & XQC_FRAME_BIT_STREAMS_BLOCKED)
+            || (po->po_frame_types & XQC_FRAME_BIT_CONNECTION_CLOSE))
         && !(po->po_flag & XQC_POF_NOT_REINJECT)
         && !(XQC_MP_PKT_REINJECTED(po))
         && (po->po_flag & XQC_POF_IN_FLIGHT)) 
diff --git a/src/transport/reinjection_control/xqc_reinj_dgram.c b/src/transport/reinjection_control/xqc_reinj_dgram.c
index dba910e9b..f770d3d10 100644
--- a/src/transport/reinjection_control/xqc_reinj_dgram.c
+++ b/src/transport/reinjection_control/xqc_reinj_dgram.c
@@ -48,7 +48,7 @@ xqc_dgram_reinj_can_reinject_after_send(xqc_dgram_reinj_ctl_t *rctl,
 {
     xqc_connection_t *conn = rctl->conn;
 
-    if ((po->po_frame_types & XQC_FRAME_BIT_DATAGRAM)
+    if ((po->po_frame_types & (XQC_FRAME_BIT_DATAGRAM | XQC_FRAME_BIT_CONNECTION_CLOSE))
         && !(po->po_flag & XQC_POF_NOT_REINJECT)
         && !(XQC_MP_PKT_REINJECTED(po))
         && (po->po_flag & XQC_POF_IN_FLIGHT)) 
diff --git a/src/transport/scheduler/xqc_scheduler_backup.c b/src/transport/scheduler/xqc_scheduler_backup.c
index a6950779e..ec7fad53c 100644
--- a/src/transport/scheduler/xqc_scheduler_backup.c
+++ b/src/transport/scheduler/xqc_scheduler_backup.c
@@ -22,47 +22,48 @@ xqc_backup_scheduler_init(void *scheduler, xqc_log_t *log, xqc_scheduler_params_
 
 
 xqc_path_ctx_t *
-xqc_backup_scheduler_get_path(void *scheduler,
-    xqc_connection_t *conn, xqc_packet_out_t *packet_out, int check_cwnd, int reinject,
+xqc_backup_scheduler_get_path(void *scheduler, xqc_connection_t *conn, 
+    xqc_packet_out_t *packet_out, int check_cwnd, int reinject, 
     xqc_bool_t *cc_blocked)
 {
-    xqc_path_ctx_t *best_path = NULL;
-    xqc_path_ctx_t *best_standby_path = NULL;
+    xqc_path_ctx_t *best_path[XQC_PATH_CLASS_PERF_CLASS_SIZE];
+    xqc_bool_t has_path[XQC_PATH_CLASS_PERF_CLASS_SIZE];
+    xqc_path_perf_class_t path_class;
+    xqc_bool_t available_path_exists;
 
     xqc_list_head_t *pos, *next;
     xqc_path_ctx_t *path;
     xqc_send_ctl_t *send_ctl;
+    xqc_bool_t path_can_send;
 
     /* min RTT */
-    uint64_t min_rtt = XQC_MAX_UINT64_VALUE;
-    uint64_t min_rtt_standby = XQC_MAX_UINT64_VALUE;
     uint64_t path_srtt;
-    uint32_t avail_path_cnt = 0;
     xqc_bool_t reached_cwnd_check = XQC_FALSE;
 
     if (cc_blocked) {
         *cc_blocked = XQC_FALSE;
     }
 
+    for (path_class = XQC_PATH_CLASS_AVAILABLE_HIGH; 
+         path_class < XQC_PATH_CLASS_PERF_CLASS_SIZE; 
+         path_class++)
+    {
+        best_path[path_class] = NULL;
+        has_path[path_class] = XQC_FALSE;
+    }
+
     xqc_list_for_each_safe(pos, next, &conn->conn_paths_list) {
         path = xqc_list_entry(pos, xqc_path_ctx_t, path_list);
 
-        /* skip inactive paths */
-        if (path->path_state != XQC_PATH_STATE_ACTIVE) {
-            continue;
-        }
+        path_class = xqc_path_get_perf_class(path);
 
+        /* skip inactive paths */
         /* skip frozen paths */
-        if (path->app_path_status == XQC_APP_PATH_STATUS_FROZEN) {
-            continue;
-        }
-
-        if (path->app_path_status == XQC_APP_PATH_STATUS_AVAILABLE) {
-            avail_path_cnt++;
-        }
-
-        if (reinject && (packet_out->po_path_id == path->path_id)) {
-            continue;
+        if (path->path_state != XQC_PATH_STATE_ACTIVE
+            || path->app_path_status == XQC_APP_PATH_STATUS_FROZEN
+            || (reinject && (packet_out->po_path_id == path->path_id))) 
+        {
+            goto skip_path;
         }
 
         if (!reached_cwnd_check) {
@@ -72,8 +73,11 @@ xqc_backup_scheduler_get_path(void *scheduler,
             }
         }
 
-        if (!xqc_scheduler_check_path_can_send(path, packet_out, check_cwnd)) {
-            continue;
+        has_path[path_class] = XQC_TRUE;
+        path_can_send = xqc_scheduler_check_path_can_send(path, packet_out, check_cwnd);
+
+        if (!path_can_send) {
+            goto skip_path;
         }
 
         if (cc_blocked) {
@@ -81,41 +85,62 @@ xqc_backup_scheduler_get_path(void *scheduler,
         }
 
         path_srtt = xqc_send_ctl_get_srtt(path->path_send_ctl);
-        xqc_log(conn->log, XQC_LOG_DEBUG, "|path srtt|conn:%p|path_id:%ui|path_srtt:%ui|", 
-                conn, path->path_id, path_srtt);
-
-        if (path_srtt < min_rtt) {
-            if (path->app_path_status == XQC_APP_PATH_STATUS_AVAILABLE) {
-                best_path = path;
-                min_rtt = path_srtt;
-                
-            } else {
-                best_standby_path = path;
-                min_rtt_standby = path_srtt;
-            }
+        
+        if (best_path[path_class] == NULL 
+            || path_srtt < best_path[path_class]->path_send_ctl->ctl_srtt)
+        {
+            best_path[path_class] = path;
         }
-    }
 
+skip_path:
+        xqc_log(conn->log, XQC_LOG_DEBUG, 
+                "|path srtt|conn:%p|path_id:%ui|path_srtt:%ui|path_class:%d|"
+                "can_send:%d|path_status:%d|path_state:%d|reinj:%d|"
+                "pkt_path_id:%ui|best_path:%i|", 
+                conn, path->path_id, path_srtt, path_class, path_can_send,
+                path->app_path_status, path->path_state, reinject, 
+                packet_out->po_path_id,
+                best_path[path_class] ? best_path[path_class]->path_id : -1);
+    }
 
-    if (best_path == NULL) {
-        if (best_standby_path != NULL
-            && (avail_path_cnt == 0 || reinject)) 
-        {
-            best_path = best_standby_path;
+    available_path_exists = XQC_FALSE;
 
-        } else {
-            xqc_log(conn->log, XQC_LOG_DEBUG, "|No available paths to schedule|conn:%p|", conn);
+    for (path_class = XQC_PATH_CLASS_AVAILABLE_HIGH; 
+         path_class < XQC_PATH_CLASS_PERF_CLASS_SIZE; 
+         path_class++)
+    {
+        if (has_path[path_class] && ((path_class & 1) == 0)) {
+            available_path_exists = XQC_TRUE;
         }
-    }
 
+        /* 
+         * do not use a standby path if there 
+         * is an available path with higher performence level
+         */
+        if (best_path[path_class] != NULL) {
+
+            if (available_path_exists && (path_class & 1) && !reinject) {
+                /* skip standby path*/
+                xqc_log(conn->log, XQC_LOG_DEBUG, 
+                        "|skip_standby_path|path_class:%d|path_id:%ui|",
+                        path_class, best_path[path_class]->path_id);
+                continue;
 
-    if (best_path) {
-        xqc_log(conn->log, XQC_LOG_DEBUG, "|best path:%ui|frame_type:%s|app_status:%d|",
-                best_path->path_id, xqc_frame_type_2_str(packet_out->po_frame_types),
-                best_path->app_path_status);
+            } else {
+                xqc_log(conn->log, XQC_LOG_DEBUG, "|best path:%ui|frame_type:%s|"
+                        "pn:%ui|size:%ud|reinj:%d|path_class:%d|",
+                        best_path[path_class]->path_id, 
+                        xqc_frame_type_2_str(packet_out->po_frame_types),
+                        packet_out->po_pkt.pkt_num,
+                        packet_out->po_used_size, reinject, path_class);
+                return best_path[path_class];
+            }
+        }
     }
 
-    return best_path;
+    xqc_log(conn->log, XQC_LOG_DEBUG, 
+            "|No available paths to schedule|conn:%p|", conn);
+    return NULL;
 }
 
 void 
diff --git a/src/transport/scheduler/xqc_scheduler_minrtt.c b/src/transport/scheduler/xqc_scheduler_minrtt.c
index ea2783734..6eef19829 100644
--- a/src/transport/scheduler/xqc_scheduler_minrtt.c
+++ b/src/transport/scheduler/xqc_scheduler_minrtt.c
@@ -25,16 +25,24 @@ xqc_minrtt_scheduler_get_path(void *scheduler,
     xqc_connection_t *conn, xqc_packet_out_t *packet_out, int check_cwnd, int reinject,
     xqc_bool_t *cc_blocked)
 {
-    xqc_path_ctx_t *best_path = NULL;
+    xqc_path_ctx_t *best_path[XQC_PATH_CLASS_PERF_CLASS_SIZE];
+    xqc_path_perf_class_t path_class;
 
     xqc_list_head_t *pos, *next;
     xqc_path_ctx_t *path;
     xqc_send_ctl_t *send_ctl;
 
     /* min RTT */
-    uint64_t min_rtt = XQC_MAX_UINT64_VALUE;
     uint64_t path_srtt;
     xqc_bool_t reached_cwnd_check = XQC_FALSE;
+    xqc_bool_t path_can_send;
+
+    for (path_class = XQC_PATH_CLASS_AVAILABLE_HIGH; 
+         path_class < XQC_PATH_CLASS_PERF_CLASS_SIZE; 
+         path_class++)
+    {
+        best_path[path_class] = NULL;
+    }
     
     if (cc_blocked) {
         *cc_blocked = XQC_FALSE;
@@ -43,17 +51,15 @@ xqc_minrtt_scheduler_get_path(void *scheduler,
     xqc_list_for_each_safe(pos, next, &conn->conn_paths_list) {
         path = xqc_list_entry(pos, xqc_path_ctx_t, path_list);
 
-        if (path->path_state != XQC_PATH_STATE_ACTIVE) {
-            continue;
-        }
-
-        /* skip the frozen path */
-        if (path->app_path_status == XQC_APP_PATH_STATUS_FROZEN) {
-            continue;
-        }
-
-        if (reinject && (packet_out->po_path_id == path->path_id)) {
-            continue;
+        path_class = xqc_path_get_perf_class(path);
+        
+        /* skip inactive paths */
+        /* skip frozen paths */
+        if (path->path_state != XQC_PATH_STATE_ACTIVE
+            || path->app_path_status == XQC_APP_PATH_STATUS_FROZEN
+            || (reinject && (packet_out->po_path_id == path->path_id)))
+        {
+            goto skip_path;
         }
 
         if (!reached_cwnd_check) {
@@ -64,12 +70,14 @@ xqc_minrtt_scheduler_get_path(void *scheduler,
         }
 
         /* @TODO: It is not correct for BBR/BBRv2, as they do not used cwnd to decide
-        *        how much data can be sent in one RTT. But, currently, BBR does not 
-        *        work well for MPQUIC due to the problem of applimit. We may adapt this
-        *        to BBR in the future, if we manage to fix the applimit problem of BBR. 
-        */
-        if (!xqc_scheduler_check_path_can_send(path, packet_out, check_cwnd)) {
-            continue;
+         *        how much data can be sent in one RTT. But, currently, BBR does not 
+         *        work well for MPQUIC due to the problem of applimit. We may adapt this
+         *        to BBR in the future, if we manage to fix the applimit problem of BBR. 
+         */
+        path_can_send = xqc_scheduler_check_path_can_send(path, packet_out, check_cwnd);
+
+        if (!path_can_send) {
+            goto skip_path;
         }
 
         if (cc_blocked) {
@@ -77,24 +85,42 @@ xqc_minrtt_scheduler_get_path(void *scheduler,
         }
 
         path_srtt = xqc_send_ctl_get_srtt(path->path_send_ctl);
-        xqc_log(conn->log, XQC_LOG_DEBUG, "|path srtt|conn:%p|path_id:%ui|path_srtt:%ui|", 
-                conn, path->path_id, path_srtt);
-
-        if (path_srtt < min_rtt) {
-            best_path = path;
-            min_rtt = path_srtt;
+        
+        if (best_path[path_class] == NULL 
+            || path_srtt < best_path[path_class]->path_send_ctl->ctl_srtt)
+        {
+            best_path[path_class] = path;
         }
+
+skip_path:
+        xqc_log(conn->log, XQC_LOG_DEBUG, 
+                "|path srtt|conn:%p|path_id:%ui|path_srtt:%ui|path_class:%d|"
+                "can_send:%d|path_status:%d|path_state:%d|reinj:%d|"
+                "pkt_path_id:%ui|best_path:%i|", 
+                conn, path->path_id, path_srtt, path_class, path_can_send,
+                path->app_path_status, path->path_state, reinject, 
+                packet_out->po_path_id,
+                best_path[path_class] ? best_path[path_class]->path_id : -1);
     }
 
-    if (best_path == NULL) {
-        xqc_log(conn->log, XQC_LOG_DEBUG, "|No available paths to schedule|conn:%p|", conn);
 
-    } else {
-        xqc_log(conn->log, XQC_LOG_DEBUG, "|best path:%ui|frame_type:%s|",
-                best_path->path_id, xqc_frame_type_2_str(packet_out->po_frame_types));
+    for (path_class = XQC_PATH_CLASS_AVAILABLE_HIGH; 
+         path_class < XQC_PATH_CLASS_PERF_CLASS_SIZE; 
+         path_class++)
+    {
+        if (best_path[path_class] != NULL) {
+            xqc_log(conn->log, XQC_LOG_DEBUG, "|best path:%ui|frame_type:%s|"
+                    "pn:%ui|size:%ud|reinj:%d|path_class:%d|",
+                    best_path[path_class]->path_id, 
+                    xqc_frame_type_2_str(packet_out->po_frame_types),
+                    packet_out->po_pkt.pkt_num,
+                    packet_out->po_used_size, reinject, path_class);
+            return best_path[path_class];
+        }
     }
 
-    return best_path;
+    xqc_log(conn->log, XQC_LOG_DEBUG, "|No available paths to schedule|conn:%p|", conn);
+    return NULL;
 }
 
 const xqc_scheduler_callback_t xqc_minrtt_scheduler_cb = {
diff --git a/src/transport/xqc_conn.c b/src/transport/xqc_conn.c
index c83a66033..2ce74cf73 100644
--- a/src/transport/xqc_conn.c
+++ b/src/transport/xqc_conn.c
@@ -70,7 +70,15 @@ xqc_conn_settings_t default_conn_settings = {
 
     .recv_rate_bytes_per_sec    = 0,
     .enable_stream_rate_limit   = 0,
-    
+
+    .scheduler_params           = {
+                                    .bw_Bps_thr = 375000, 
+                                    .loss_percent_thr_high = 30,
+                                    .loss_percent_thr_low = 10,
+                                    .pto_cnt_thr = 2,
+                                    .rtt_us_thr_high = 2000000,
+                                    .rtt_us_thr_low = 500000
+                                  },
     .is_interop_mode            = 0,
 };
 
@@ -212,9 +220,6 @@ static const char * const xqc_conn_flag_to_str[XQC_CONN_FLAG_SHIFT_NUM] = {
     [XQC_CONN_FLAG_HANDSHAKE_COMPLETED_SHIFT]   = "HSK_DONE",
     [XQC_CONN_FLAG_CAN_SEND_1RTT_SHIFT]         = "CAN_SEND_1RTT",
     [XQC_CONN_FLAG_TICKING_SHIFT]               = "TICKING",
-    [XQC_CONN_FLAG_SHOULD_ACK_INIT_SHIFT]       = "ACK_INIT",
-    [XQC_CONN_FLAG_SHOULD_ACK_HSK_SHIFT]        = "ACK_HSK",
-    [XQC_CONN_FLAG_SHOULD_ACK_01RTT_SHIFT]      = "ACK_01RTT",
     [XQC_CONN_FLAG_ACK_HAS_GAP_SHIFT]           = "HAS_GAP",
     [XQC_CONN_FLAG_TIME_OUT_SHIFT]              = "TIME_OUT",
     [XQC_CONN_FLAG_ERROR_SHIFT]                 = "ERROR",
@@ -470,6 +475,10 @@ xqc_conn_create(xqc_engine_t *engine, xqc_cid_t *dcid, xqc_cid_t *scid,
         xc->conn_settings.datagram_redundancy = XQC_MAX_DATAGRAM_REDUNDANCY;
     }
 
+    if (xc->conn_settings.mp_enable_reinjection & XQC_REINJ_UNACK_BEFORE_SCHED) {
+        xc->conn_settings.mp_enable_reinjection |= XQC_REINJ_UNACK_AFTER_SEND;
+    }
+
     if (xc->conn_settings.datagram_redundancy) {
         if (xc->conn_settings.datagram_redundancy == 1) {
             /* reinject packets on any path */
@@ -679,10 +688,6 @@ xqc_conn_create(xqc_engine_t *engine, xqc_cid_t *dcid, xqc_cid_t *scid,
         goto fail;
     }
 
-    if (xqc_conn_init_paths_list(xc) != XQC_OK) {
-        goto fail;
-    }
-
     /* set scheduler callback (default: minRTT) */
     if (xc->conn_settings.scheduler_callback.xqc_scheduler_init) {
         xc->scheduler_callback = &xc->conn_settings.scheduler_callback;
@@ -692,6 +697,9 @@ xqc_conn_create(xqc_engine_t *engine, xqc_cid_t *dcid, xqc_cid_t *scid,
     }
 
     xc->scheduler = xqc_pcalloc(xc->conn_pool, xc->scheduler_callback->xqc_scheduler_size());
+    if (xc->scheduler == NULL) {
+        goto fail;
+    }
     xc->scheduler_callback->xqc_scheduler_init(xc->scheduler, xc->log, &xc->conn_settings.scheduler_params);
 
     /* set reinject control callback if reinjection enabled */
@@ -704,6 +712,13 @@ xqc_conn_create(xqc_engine_t *engine, xqc_cid_t *dcid, xqc_cid_t *scid,
         xc->reinj_callback->xqc_reinj_ctl_init(xc->reinj_ctl, xc);
     }
 
+    /* 
+     * Init paths after the scheduler and the reinjection controller are initialized. 
+     */
+    if (xqc_conn_init_paths_list(xc) != XQC_OK) {
+        goto fail;
+    }
+
     xc->pkt_filter_cb = NULL;
 
     /* for datagram */
@@ -983,6 +998,7 @@ xqc_conn_destroy(xqc_connection_t *xc)
     }
 
     xqc_conn_stats_t conn_stats;
+    xqc_memzero(&conn_stats, sizeof(xqc_conn_stats_t));
     xqc_conn_get_stats_internal(xc, &conn_stats);
 
     if (xc->tls) {
@@ -998,7 +1014,8 @@ xqc_conn_destroy(xqc_connection_t *xc)
             "has_0rtt:%d|0rtt_accept:%d|token_ok:%d|handshake_time:%ui|"
             "first_send_delay:%ui|conn_persist:%ui|keyupdate_cnt:%d|err:0x%xi|close_msg:%s|%s|"
             "hsk_recv:%ui|close_recv:%ui|close_send:%ui|last_recv:%ui|last_send:%ui|"
-            "mp_enable:%ud|create:%ud|validated:%ud|active:%ud|path_info:%s|alpn:%*s",
+            "mp_enable:%ud|create:%ud|validated:%ud|active:%ud|path_info:%s|alpn:%*s|rebind_count:%d|"
+            "rebind_valid:%d|",
             xc,
             xc->conn_flag & XQC_CONN_FLAG_HAS_0RTT ? 1:0,
             xc->conn_flag & XQC_CONN_FLAG_0RTT_OK ? 1:0,
@@ -1013,7 +1030,8 @@ xqc_conn_destroy(xqc_connection_t *xc)
             xqc_calc_delay(xc->conn_last_recv_time, xc->conn_create_time),
             xqc_calc_delay(xc->conn_last_send_time, xc->conn_create_time),
             xc->enable_multipath, xc->create_path_count, xc->validated_path_count, xc->active_path_count,
-            conn_stats.conn_info, out_alpn_len, out_alpn);
+            conn_stats.conn_info, out_alpn_len, out_alpn, conn_stats.total_rebind_count,
+            conn_stats.total_rebind_valid);
     xqc_log_event(xc->log, CON_CONNECTION_CLOSED, xc);
 
     if (xc->conn_flag & XQC_CONN_FLAG_WAIT_WAKEUP) {
@@ -1369,12 +1387,8 @@ xqc_conn_schedule_packets(xqc_connection_t *conn,  xqc_list_head_t *head,
     xqc_list_for_each_safe(pos, next, head) {
         packet_out = xqc_list_entry(pos, xqc_packet_out_t, po_list);
 
-        /* 1. connection level not support schedule multipath */
-        if (conn->enable_multipath == XQC_CONN_NOT_SUPPORT_MULTIPATH) {
-            path = conn->conn_initial_path;
-
-        /* 2. 已设置特定路径发送的包，例如：PATH_CHALLENGE PATH_RESPONSE MP_ACK(原路径ACK) */
-        } else if (xqc_packet_out_on_specific_path(conn, packet_out, &path)) {
+        /* 1. 已设置特定路径发送的包，例如：PATH_CHALLENGE PATH_RESPONSE MP_ACK(原路径ACK) */
+        if (xqc_packet_out_on_specific_path(conn, packet_out, &path)) {
             
             if (path == NULL) {
                 continue;
@@ -1383,7 +1397,7 @@ xqc_conn_schedule_packets(xqc_connection_t *conn,  xqc_list_head_t *head,
                     path->path_id, path->path_state, xqc_frame_type_2_str(packet_out->po_frame_types),
                     packet_out->po_stream_id, packet_out->po_stream_offset);
 
-        /* 3. schedule packet multipath */
+        /* 2. schedule packet multipath */
         } else {
             path = conn->scheduler_callback->
                    xqc_scheduler_get_path(conn->scheduler, 
@@ -1820,7 +1834,7 @@ xqc_conn_enc_packet(xqc_connection_t *conn,
         conn->conn_state = XQC_CONN_STATE_CLOSED;
         return -XQC_EENCRYPT;
     }
-    
+
     packet_out->po_sent_time = current_time;
     return XQC_OK;
 }
@@ -2726,6 +2740,47 @@ xqc_conn_continue_send(xqc_engine_t *engine, const xqc_cid_t *cid)
     return XQC_OK;
 }
 
+void 
+xqc_conn_encode_mp_settings(xqc_connection_t *conn, char *buf, size_t buf_sz)
+{
+    size_t len = 0;
+    uint8_t encode_val = 0;
+
+    if (buf_sz < XQC_MP_SETTINGS_STR_LEN) {
+        return;
+    }
+
+    if (conn->enable_multipath) {
+        encode_val = 1;
+    }
+    //len: 1
+    len = snprintf(buf, buf_sz, "%d", encode_val);
+
+    encode_val = 0;
+    if (conn->local_settings.enable_multipath) {
+        encode_val = 1;
+    }
+    //len: 3
+    len += snprintf(buf + len, buf_sz - len, "/%d", encode_val);
+
+    encode_val = 0;
+    if (conn->remote_settings.enable_multipath) {
+        encode_val = 1;
+    }
+    //len: 5
+    len += snprintf(buf + len, buf_sz - len, "/%d", encode_val);
+
+    //len: 9
+    encode_val = conn->local_settings.multipath_version;
+    len += snprintf(buf + len, buf_sz - len, "/%d", encode_val);
+
+    //len: 13
+    encode_val = conn->remote_settings.multipath_version;
+    len += snprintf(buf + len, buf_sz - len, "/%d", encode_val);
+
+    buf[len] = 0;
+}
+
 void
 xqc_conn_info_print(xqc_connection_t *conn, xqc_conn_stats_t *conn_stats)
 {
@@ -2738,35 +2793,13 @@ xqc_conn_info_print(xqc_connection_t *conn, xqc_conn_stats_t *conn_stats)
     xqc_list_head_t *pos, *next;
     xqc_path_ctx_t *path = NULL;
     xqc_path_info_t path_info;
-    uint32_t mp_settings = 0;
+    char mp_settings[XQC_MP_SETTINGS_STR_LEN] = {0};
     uint32_t sock_err_flag = 0;
 
-    if (conn->enable_multipath) {
-        mp_settings |= 1;
-    }
-
-    if (conn->local_settings.enable_multipath) {
-        mp_settings |= (1 << 1);
-    }
-
-    if (conn->remote_settings.enable_multipath) {
-        mp_settings |= (1 << 2);
-    }
-
-    if (conn->conn_settings.multipath_version == XQC_MULTIPATH_05) {
-        mp_settings |= (1 << 3);
-    }
-
-    if (conn->local_settings.multipath_version == XQC_MULTIPATH_05) {
-        mp_settings |= (1 << 4);
-    }
-
-    if (conn->remote_settings.multipath_version == XQC_MULTIPATH_05) {
-        mp_settings |= (1 << 5);
-    }
+    xqc_conn_encode_mp_settings(conn, mp_settings, XQC_MP_SETTINGS_STR_LEN);
 
     /* conn info */
-    ret = snprintf(buff, buff_size, "%u,%u,%u,%u,%u,%u,%u,"
+    ret = snprintf(buff, buff_size, "%s,%u,%u,%u,%u,%u,%u,"
                    "%u,%u,%u,%u,%u,%u,%u,%"PRIu64",%"PRIu64",%"PRIu64",",
                    mp_settings,
                    conn->create_path_count,
@@ -2796,15 +2829,15 @@ xqc_conn_info_print(xqc_connection_t *conn, xqc_conn_stats_t *conn_stats)
     for (i = 0; i < 3; i++) {
 
         ret = snprintf(buff + curr_size, buff_size - curr_size,
-                       "%u,%u,%u,%"PRIu64",%"PRIu64","
-                       "%"PRIu64",%"PRIu64",",
+                       "%u,%u,%u,%"PRIx64",%"PRIu64","
+                       "%"PRIu64",%d,",
                        (uint32_t)conn->rcv_pkt_stats.pkt_types[i],
                        conn->rcv_pkt_stats.pkt_size[i],
                        conn->rcv_pkt_stats.pkt_udp_size[i],
                        (uint64_t)conn->rcv_pkt_stats.pkt_frames[i],
                        (uint64_t)conn->rcv_pkt_stats.pkt_pn[i],
                        (uint64_t)conn->rcv_pkt_stats.pkt_timestamp[i],
-                       (uint64_t)conn->rcv_pkt_stats.pkt_err[i]);
+                       conn->rcv_pkt_stats.pkt_err[i]);
 
         curr_size += ret;
 
@@ -2816,7 +2849,7 @@ xqc_conn_info_print(xqc_connection_t *conn, xqc_conn_stats_t *conn_stats)
     /* send_stats */
     for (i = 0; i < 3; i++) {
         ret = snprintf(buff + curr_size, buff_size - curr_size,
-                       "%u,%u,%"PRIu64",%"PRIu64","
+                       "%u,%u,%"PRIx64",%"PRIu64","
                        "%"PRIu64",",
                        (uint32_t)conn->snd_pkt_stats.pkt_types[i],
                        conn->snd_pkt_stats.pkt_size[i],
@@ -2846,13 +2879,14 @@ xqc_conn_info_print(xqc_connection_t *conn, xqc_conn_stats_t *conn_stats)
                        "%d-%u,%d-%u,"
                        "%d-%u,%d-%u,"
                        "%d-%u,%d-%u,"
+                       "%d-%u,%d-%u,"
                        "%d-%u,%d-%u,",
                        (int)path_info.path_id, path_info.path_state,
                        (int)path_info.path_id, path_info.app_path_status,
                        (int)path_info.path_id, sock_err_flag,
                        (int)path_info.path_id, path_info.path_create_time,
                        (int)path_info.path_id, path_info.path_destroy_time,
-                       (int)path_info.path_id, path_info.srtt,
+                       (int)path_info.path_id, path_info.srtt / 1000,
                        (int)path_info.path_id, path_info.path_bytes_send,
                        (int)path_info.path_id, path_info.path_bytes_recv,
                        (int)path_info.path_id, path_info.pkt_send_cnt,
@@ -2862,7 +2896,9 @@ xqc_conn_info_print(xqc_connection_t *conn, xqc_conn_stats_t *conn_stats)
                        (int)path_info.path_id, path_info.dgram_send_cnt,
                        (int)path_info.path_id, path_info.dgram_recv_cnt,
                        (int)path_info.path_id, path_info.red_dgram_send_cnt,
-                       (int)path_info.path_id, path_info.red_dgram_recv_cnt);
+                       (int)path_info.path_id, path_info.red_dgram_recv_cnt,
+                       (int)path_info.path_id, path->rebinding_count,
+                       (int)path_info.path_id, path->rebinding_valid);
 
         curr_size += ret;
 
@@ -2958,6 +2994,8 @@ xqc_conn_get_stats_internal(xqc_connection_t *conn, xqc_conn_stats_t *conn_stats
         conn_stats->recv_count           += send_ctl->ctl_recv_count;
         conn_stats->lost_dgram_count     += send_ctl->ctl_lost_dgram_cnt;
         conn_stats->inflight_bytes       += send_ctl->ctl_bytes_in_flight;
+        conn_stats->total_rebind_count   += path->rebinding_count;
+        conn_stats->total_rebind_valid   += path->rebinding_valid;
     }
 
     /* 路径信息 */
@@ -3419,6 +3457,15 @@ xqc_conn_process_undecrypt_packet_in(xqc_connection_t *conn, xqc_encrypt_level_t
     return XQC_OK;
 }
 
+void 
+xqc_conn_buff_1rtt_packet(xqc_connection_t *conn, xqc_packet_out_t *po)
+{
+    xqc_send_queue_remove_send(&po->po_list);
+    xqc_send_queue_insert_buff(&po->po_list, &conn->conn_send_queue->sndq_buff_1rtt_packets);
+    if (!(conn->conn_flag & XQC_CONN_FLAG_DCID_OK)) {
+        po->po_flag |= XQC_POF_DCID_NOT_DONE;
+    }
+}
 
 void
 xqc_conn_buff_1rtt_packets(xqc_connection_t *conn)
diff --git a/src/transport/xqc_conn.h b/src/transport/xqc_conn.h
index 2700a8d6d..86d843987 100644
--- a/src/transport/xqc_conn.h
+++ b/src/transport/xqc_conn.h
@@ -36,6 +36,8 @@
 
 #define XQC_MAX_RECV_WINDOW (16 * 1024 * 1024)
 
+#define XQC_MP_SETTINGS_STR_LEN (30)
+
 static const uint32_t MAX_RSP_CONN_CLOSE_CNT = 3;
 
 /* for debugging, will be deleted later */
@@ -87,10 +89,6 @@ typedef enum {
     XQC_CONN_STATE_N,
 } xqc_conn_state_t;
 
-#define XQC_CONN_FLAG_SHOULD_ACK (XQC_CONN_FLAG_SHOULD_ACK_INIT     \
-                                  | XQC_CONN_FLAG_SHOULD_ACK_HSK    \
-                                  | XQC_CONN_FLAG_SHOULD_ACK_01RTT) \
-
 #define XQC_CONN_IMMEDIATE_CLOSE_FLAGS (XQC_CONN_FLAG_ERROR)
 
 /* !!WARNING: to add flag, please update conn_flag_2_str */
@@ -99,9 +97,6 @@ typedef enum {
     XQC_CONN_FLAG_HANDSHAKE_COMPLETED_SHIFT,
     XQC_CONN_FLAG_CAN_SEND_1RTT_SHIFT,
     XQC_CONN_FLAG_TICKING_SHIFT,
-    XQC_CONN_FLAG_SHOULD_ACK_INIT_SHIFT,
-    XQC_CONN_FLAG_SHOULD_ACK_HSK_SHIFT      = (XQC_CONN_FLAG_SHOULD_ACK_INIT_SHIFT + XQC_PNS_HSK),
-    XQC_CONN_FLAG_SHOULD_ACK_01RTT_SHIFT    = (XQC_CONN_FLAG_SHOULD_ACK_INIT_SHIFT + XQC_PNS_APP_DATA),
     XQC_CONN_FLAG_ACK_HAS_GAP_SHIFT,
     XQC_CONN_FLAG_TIME_OUT_SHIFT,
     XQC_CONN_FLAG_ERROR_SHIFT,
@@ -147,9 +142,6 @@ typedef enum {
     XQC_CONN_FLAG_HANDSHAKE_COMPLETED   = 1ULL << XQC_CONN_FLAG_HANDSHAKE_COMPLETED_SHIFT,
     XQC_CONN_FLAG_CAN_SEND_1RTT         = 1ULL << XQC_CONN_FLAG_CAN_SEND_1RTT_SHIFT,
     XQC_CONN_FLAG_TICKING               = 1ULL << XQC_CONN_FLAG_TICKING_SHIFT,
-    XQC_CONN_FLAG_SHOULD_ACK_INIT       = 1ULL << XQC_CONN_FLAG_SHOULD_ACK_INIT_SHIFT,
-    XQC_CONN_FLAG_SHOULD_ACK_HSK        = 1ULL << XQC_CONN_FLAG_SHOULD_ACK_HSK_SHIFT,
-    XQC_CONN_FLAG_SHOULD_ACK_01RTT      = 1ULL << XQC_CONN_FLAG_SHOULD_ACK_01RTT_SHIFT,
     XQC_CONN_FLAG_ACK_HAS_GAP           = 1ULL << XQC_CONN_FLAG_ACK_HAS_GAP_SHIFT,
     XQC_CONN_FLAG_TIME_OUT              = 1ULL << XQC_CONN_FLAG_TIME_OUT_SHIFT,
     XQC_CONN_FLAG_ERROR                 = 1ULL << XQC_CONN_FLAG_ERROR_SHIFT,
@@ -313,6 +305,9 @@ struct xqc_connection_s {
 
     xqc_trans_settings_t            local_settings;
     xqc_trans_settings_t            remote_settings;
+
+    /* a bitmap to record if ACKs should be generated for path[pns] */
+    uint64_t                        ack_flag;
     xqc_conn_flag_t                 conn_flag;
     xqc_conn_type_t                 conn_type;
 
@@ -426,7 +421,7 @@ struct xqc_connection_s {
         xqc_frame_type_bit_t        pkt_frames[3];
         uint32_t                    pkt_size[3];
         uint32_t                    pkt_udp_size[3];
-        uint64_t                    pkt_err[3];
+        int                         pkt_err[3];
         xqc_usec_t                  pkt_timestamp[3];
         xqc_packet_number_t         pkt_pn[3];
         uint8_t                     curr_index;
@@ -496,6 +491,7 @@ xqc_int_t xqc_conn_handshake_complete(xqc_connection_t *conn);
 xqc_int_t xqc_conn_buff_undecrypt_packet_in(xqc_packet_in_t *packet_in, xqc_connection_t *conn,
     xqc_encrypt_level_t encrypt_level);
 xqc_int_t xqc_conn_process_undecrypt_packet_in(xqc_connection_t *conn, xqc_encrypt_level_t encrypt_level);
+void xqc_conn_buff_1rtt_packet(xqc_connection_t *conn, xqc_packet_out_t *po);
 void xqc_conn_buff_1rtt_packets(xqc_connection_t *conn);
 void xqc_conn_write_buffed_1rtt_packets(xqc_connection_t *conn);
 xqc_usec_t xqc_conn_next_wakeup_time(xqc_connection_t *conn);
@@ -538,17 +534,6 @@ xqc_conn_has_undecrypt_packets(xqc_connection_t *conn)
         || conn->undecrypt_count[XQC_ENC_LEV_HSK];
 }
 
-static inline xqc_int_t
-xqc_conn_should_ack(xqc_connection_t *conn)
-{
-    if (conn->conn_flag & XQC_CONN_FLAG_SHOULD_ACK) {
-        xqc_log(conn->log, XQC_LOG_DEBUG, "|should_generate_ack yes|flag:%s|",
-                xqc_conn_flag_2_str(conn->conn_flag));
-        return 1;
-    }
-    return 0;
-}
-
 /* process an UDP datagram */
 #ifdef XQC_NO_PID_PACKET_PROCESS
 xqc_int_t xqc_conn_process_packet(xqc_connection_t *c, const unsigned char *packet_in_buf,
@@ -666,4 +651,6 @@ void xqc_conn_destroy_ping_notification_list(xqc_connection_t *conn);
 
 xqc_int_t xqc_conn_send_ping_internal(xqc_connection_t *conn, void *ping_user_data, xqc_bool_t notify);
 
+void xqc_conn_encode_mp_settings(xqc_connection_t *conn, char *buf, size_t buf_sz);
+
 #endif /* _XQC_CONN_H_INCLUDED_ */
diff --git a/src/transport/xqc_engine.c b/src/transport/xqc_engine.c
index 779d880e7..f35b052bf 100644
--- a/src/transport/xqc_engine.c
+++ b/src/transport/xqc_engine.c
@@ -745,13 +745,14 @@ xqc_engine_process_conn(xqc_connection_t *conn, xqc_usec_t now)
     }
     XQC_CHECK_IMMEDIATE_CLOSE();
 
-    if (xqc_conn_should_ack(conn)) {
+    if (conn->ack_flag) {
         ret = xqc_write_ack_or_mp_ack_to_packets(conn);
         if (ret) {
             xqc_log(conn->log, XQC_LOG_ERROR, "|xqc_write_ack_or_mp_ack_to_packets error|");
             XQC_CONN_ERR(conn, TRA_INTERNAL_ERROR);
         }
     }
+    
     XQC_CHECK_IMMEDIATE_CLOSE();
 
 
@@ -1127,6 +1128,8 @@ xqc_engine_packet_process(xqc_engine_t *engine,
     xqc_int_t ret;
     xqc_connection_t *conn = NULL;
     xqc_cid_t dcid, scid;   /* dcid: cid of peer; scid: cid of endpoint */
+    xqc_log_level_t lvl;
+
     xqc_cid_init_zero(&dcid);
     xqc_cid_init_zero(&scid);
 
@@ -1180,8 +1183,16 @@ xqc_engine_packet_process(xqc_engine_t *engine,
             return -XQC_ECONN_NFOUND;
         }
 
-        xqc_log(engine->log, XQC_LOG_STATS, "|fail to find connection, send "
-                "reset|size:%uz|scid:%s|", packet_in_size, xqc_scid_str(&scid));
+        lvl = XQC_LOG_STATS;
+        if (engine->eng_type == XQC_ENGINE_CLIENT) {
+            lvl = XQC_LOG_REPORT;
+        }
+
+        xqc_log(engine->log, lvl, "|fail to find connection, send reset|"
+                "size:%uz|scid:%s|recv_time:%ui|peer_addr:%s|local_addr:%s",
+                packet_in_size, xqc_scid_str(&scid), recv_time,
+                xqc_peer_addr_str(peer_addr, peer_addrlen),
+                xqc_local_addr_str(local_addr, local_addrlen));
 
         ret = xqc_engine_send_reset(engine, &scid, peer_addr, peer_addrlen,
                                     local_addr, local_addrlen, packet_in_size,
diff --git a/src/transport/xqc_multipath.c b/src/transport/xqc_multipath.c
index 46e5dbf85..35694a4f8 100644
--- a/src/transport/xqc_multipath.c
+++ b/src/transport/xqc_multipath.c
@@ -76,6 +76,12 @@ xqc_path_create(xqc_connection_t *conn, xqc_cid_t *scid, xqc_cid_t *dcid)
 {
     xqc_path_ctx_t *path = NULL;
 
+    if (conn->create_path_count >= XQC_MAX_PATHS_COUNT) {
+        xqc_log(conn->log, XQC_LOG_ERROR, 
+                "|too many paths|current maximum:%d|", XQC_MAX_PATHS_COUNT);
+        return NULL;
+    }
+
     path = xqc_calloc(1, sizeof(xqc_path_ctx_t));
     if (path == NULL) {
         return NULL;
@@ -837,10 +843,10 @@ xqc_stream_path_metrics_on_send(xqc_connection_t *conn, xqc_packet_out_t *po)
             if (stream != NULL && po->po_path_id < XQC_MAX_PATHS_COUNT) {
                 stream->paths_info[po->po_path_id].path_id = po->po_path_id;
                 stream->paths_info[po->po_path_id].path_pkt_send_count += 1;
-                stream->paths_info[po->po_path_id].path_send_bytes += po->po_used_size;
+                stream->paths_info[po->po_path_id].path_send_bytes += po->po_stream_frames[i].ps_length;
 
                 if (po->po_flag & XQC_POF_REINJECTED_REPLICA) {
-                    stream->paths_info[po->po_path_id].path_send_reinject_bytes += po->po_used_size;
+                    stream->paths_info[po->po_path_id].path_send_reinject_bytes += po->po_stream_frames[i].ps_length;
                 }
             }
 
@@ -1466,4 +1472,71 @@ xqc_path_standby_probe(xqc_path_ctx_t *path)
     xqc_log(conn->log, XQC_LOG_DEBUG, "|PING|path:%ui|", path->path_id);
     path->standby_probe_count++;
     return XQC_OK;
+}
+
+xqc_path_perf_class_t 
+xqc_path_get_perf_class(xqc_path_ctx_t *path)
+{
+    xqc_connection_t *conn = path->parent_conn;
+    xqc_scheduler_params_t *param = &conn->conn_settings.scheduler_params;
+    xqc_usec_t path_srtt = xqc_send_ctl_get_srtt(path->path_send_ctl);
+    xqc_usec_t min_srtt = xqc_conn_get_min_srtt(path->parent_conn, 0);
+    uint64_t path_bw = xqc_send_ctl_get_est_bw(path->path_send_ctl);
+    double loss_rate = xqc_path_recent_loss_rate(path);
+
+    xqc_log(conn->log, XQC_LOG_DEBUG, "|conn:%p|path_id:%ui|"
+            "path_srtt:%ui|min_srtt:%ui|path_bw:%ui|loss_rate:%.2f|"
+            "path_pto:%ud|", 
+            conn, path->path_id, path_srtt, min_srtt, path_bw, loss_rate,
+            path->path_send_ctl->ctl_pto_count);
+
+    // low 
+    if (path_srtt > param->rtt_us_thr_high
+        || path->path_send_ctl->ctl_pto_count >= param->pto_cnt_thr
+        || loss_rate > param->loss_percent_thr_high) 
+    {
+        if (path->app_path_status == XQC_APP_PATH_STATUS_AVAILABLE) {
+            return XQC_PATH_CLASS_AVAILABLE_LOW;
+
+        } else {
+            return XQC_PATH_CLASS_STANDBY_LOW;
+        }
+    }
+
+    // mid 
+    if ((path_srtt <= param->rtt_us_thr_high && (path_srtt > xqc_min(param->rtt_us_thr_low, 3 * min_srtt)))
+        || path_bw < param->bw_Bps_thr
+        || (loss_rate <= param->loss_percent_thr_high && loss_rate > param->loss_percent_thr_low))
+    {
+        if (path->app_path_status == XQC_APP_PATH_STATUS_AVAILABLE) {
+            return XQC_PATH_CLASS_AVAILABLE_MID;
+
+        } else {
+            return XQC_PATH_CLASS_STANDBY_MID;
+        }
+    }
+
+    // high
+    if (path->app_path_status == XQC_APP_PATH_STATUS_AVAILABLE) {
+        return XQC_PATH_CLASS_AVAILABLE_HIGH;
+    }
+
+    return XQC_PATH_CLASS_STANDBY_HIGH;
+}
+
+
+double 
+xqc_path_recent_loss_rate(xqc_path_ctx_t *path)
+{
+    double loss_rate0 = 0, loss_rate1 = 0;
+    
+    if (path->path_send_ctl->ctl_recent_send_count[0]) {
+        loss_rate0 = 100.0 * path->path_send_ctl->ctl_recent_lost_count[0] / path->path_send_ctl->ctl_recent_send_count[0];
+    }
+
+    if (path->path_send_ctl->ctl_recent_send_count[1]) {
+        loss_rate1 = 100.0 * path->path_send_ctl->ctl_recent_lost_count[1] / path->path_send_ctl->ctl_recent_send_count[1];
+    }
+
+    return xqc_max(loss_rate0, loss_rate1);
 }
\ No newline at end of file
diff --git a/src/transport/xqc_multipath.h b/src/transport/xqc_multipath.h
index 7ddb220bb..5ddffa209 100644
--- a/src/transport/xqc_multipath.h
+++ b/src/transport/xqc_multipath.h
@@ -60,9 +60,21 @@ typedef enum {
 } xqc_send_type_t;
 
 typedef enum {
-    XQC_PATH_FLAG_SEND_STATUS       = 1 << 0,
-    XQC_PATH_FLAG_RECV_STATUS       = 1 << 1,
-    XQC_PATH_FLAG_SOCKET_ERROR      = 1 << 2,
+    XQC_PATH_FLAG_SEND_STATUS_SHIFT,
+    XQC_PATH_FLAG_RECV_STATUS_SHIFT,
+    XQC_PATH_FLAG_SOCKET_ERROR_SHIFT,
+    XQC_PATH_FLAG_SHOULD_ACK_INIT_SHIFT,
+    XQC_PATH_FLAG_SHOULD_ACK_HSK_SHIFT,
+    XQC_PATH_FLAG_SHOULD_ACK_01RTT_SHIFT,
+} xqc_path_flag_shift_t;
+
+typedef enum {
+    XQC_PATH_FLAG_SEND_STATUS       = 1 << XQC_PATH_FLAG_SEND_STATUS_SHIFT,
+    XQC_PATH_FLAG_RECV_STATUS       = 1 << XQC_PATH_FLAG_RECV_STATUS_SHIFT,
+    XQC_PATH_FLAG_SOCKET_ERROR      = 1 << XQC_PATH_FLAG_SOCKET_ERROR_SHIFT,
+    XQC_PATH_FLAG_SHOULD_ACK_INIT   = 1 << XQC_PATH_FLAG_SHOULD_ACK_INIT_SHIFT,
+    XQC_PATH_FLAG_SHOULD_ACK_HSK    = 1 << XQC_PATH_FLAG_SHOULD_ACK_HSK_SHIFT,
+    XQC_PATH_FLAG_SHOULD_ACK_01RTT  = 1 << XQC_PATH_FLAG_SHOULD_ACK_01RTT_SHIFT,
 } xqc_path_flag_t;
 
 typedef enum {
@@ -75,6 +87,16 @@ typedef enum {
     XQC_PATH_SPECIFIED_BY_PQP       = 1 << 6,  /* Path Quality Probe */
 } xqc_path_specified_flag_t;
 
+typedef enum {
+    XQC_PATH_CLASS_AVAILABLE_HIGH,
+    XQC_PATH_CLASS_STANDBY_HIGH,
+    XQC_PATH_CLASS_AVAILABLE_MID,
+    XQC_PATH_CLASS_STANDBY_MID,
+    XQC_PATH_CLASS_AVAILABLE_LOW,
+    XQC_PATH_CLASS_STANDBY_LOW,
+    XQC_PATH_CLASS_PERF_CLASS_SIZE,
+} xqc_path_perf_class_t;
+
 /* path context */
 struct xqc_path_ctx_s {
 
@@ -246,6 +268,10 @@ xqc_bool_t xqc_path_is_full(xqc_path_ctx_t *path);
 
 xqc_int_t xqc_path_standby_probe(xqc_path_ctx_t *path);
 
+xqc_path_perf_class_t xqc_path_get_perf_class(xqc_path_ctx_t *path);
+
+double xqc_path_recent_loss_rate(xqc_path_ctx_t *path);
+
 #endif /* XQC_MULTIPATH_H */
 
 
diff --git a/src/transport/xqc_packet_out.c b/src/transport/xqc_packet_out.c
index 73f2ac37a..f9754af05 100644
--- a/src/transport/xqc_packet_out.c
+++ b/src/transport/xqc_packet_out.c
@@ -409,7 +409,8 @@ xqc_write_ack_to_one_packet(xqc_connection_t *conn, xqc_packet_out_t *packet_out
     } else {
         conn->conn_flag &= ~XQC_CONN_FLAG_ACK_HAS_GAP;
     }
-    conn->conn_flag &= ~(XQC_CONN_FLAG_SHOULD_ACK_INIT << pns);
+    path->path_flag &= ~(XQC_PATH_FLAG_SHOULD_ACK_INIT << pns);
+    conn->ack_flag &= ~(1 << (pns + path->path_id * XQC_PNS_N));
 
     return XQC_OK;
 
@@ -439,22 +440,23 @@ xqc_write_ack_or_mp_ack_to_packets(xqc_connection_t *conn)
     xqc_pkt_type_t pkt_type;
     xqc_list_head_t *pos, *next;
     xqc_bool_t is_mp_ack = 0;       /* Send ack in default case */
-
-    int ret;
-    
+    int ret = XQC_OK;
     xqc_path_ctx_t *path;
     xqc_list_head_t *path_pos, *path_next;
     
-    for (pns = 0; pns < XQC_PNS_N; ++pns) {
-        
-        /* If there's no such pns in the connection, continue the loop */
-        if (!(conn->conn_flag & (XQC_CONN_FLAG_SHOULD_ACK_INIT << pns))) {
+    xqc_list_for_each_safe(path_pos, path_next, &conn->conn_paths_list) {
+        path = xqc_list_entry(path_pos, xqc_path_ctx_t, path_list);
+
+        if (path->path_state < XQC_PATH_STATE_VALIDATING
+            || path->path_state >= XQC_PATH_STATE_CLOSED) 
+        {
             continue;
         }
 
-        xqc_list_for_each_safe(path_pos, path_next, &conn->conn_paths_list) {
-            path = xqc_list_entry(path_pos, xqc_path_ctx_t, path_list);       
-            if (path->path_state < XQC_PATH_STATE_VALIDATING) {
+        for (pns = 0; pns < XQC_PNS_N; ++pns) {
+
+            /* If there's no such pns in the connection, continue the loop */
+            if (!(path->path_flag & (XQC_PATH_FLAG_SHOULD_ACK_INIT << pns))) {
                 continue;
             }
 
@@ -466,6 +468,7 @@ xqc_write_ack_or_mp_ack_to_packets(xqc_connection_t *conn)
                 first_range = xqc_list_entry(tmp_pos, xqc_pktno_range_node_t, list);
                 break;
             }
+            
             if (first_range == NULL) {
                 continue;
             }
@@ -541,13 +544,9 @@ xqc_write_ack_or_mp_ack_to_packets(xqc_connection_t *conn)
 
 done:
             xqc_log(conn->log, XQC_LOG_DEBUG, "|path:%ui|pns:%d|", path->path_id, pns);
-            
-            /* Ack frame should only be sent on initial path */
-            if (!is_mp_ack) {
-                break;
-            }
         }
     }
+
     return XQC_OK;
 }
 
@@ -686,8 +685,25 @@ xqc_write_reset_stream_to_packet(xqc_connection_t *conn, xqc_stream_t *stream,
 {
     ssize_t ret;
     xqc_packet_out_t *packet_out;
+    xqc_pkt_type_t pkt_type = XQC_PTYPE_SHORT_HEADER;
+    int support_0rtt = xqc_conn_is_ready_to_send_early_data(conn);
+    xqc_bool_t buff_reset = XQC_FALSE;
+
+    if (!(conn->conn_flag & XQC_CONN_FLAG_CAN_SEND_1RTT)) {
+        if ((conn->conn_type == XQC_CONN_TYPE_CLIENT) 
+            && (conn->conn_state == XQC_CONN_STATE_CLIENT_INITIAL_SENT) 
+            && support_0rtt)
+        {
+            pkt_type = XQC_PTYPE_0RTT;
+            conn->conn_flag |= XQC_CONN_FLAG_HAS_0RTT;
+            stream->stream_flag |= XQC_STREAM_FLAG_HAS_0RTT;
 
-    packet_out = xqc_write_new_packet(conn, XQC_PTYPE_NUM);
+        } else {
+            buff_reset = XQC_TRUE;
+        }
+    }
+
+    packet_out = xqc_write_new_packet(conn, pkt_type);
     if (packet_out == NULL) {
         xqc_log(conn->log, XQC_LOG_ERROR, "|xqc_write_new_packet error|");
         return -XQC_EWRITE_PKT;
@@ -715,6 +731,10 @@ xqc_write_reset_stream_to_packet(xqc_connection_t *conn, xqc_stream_t *stream,
         stream->stream_stats.app_reset_time = xqc_monotonic_timestamp();
     }
 
+    if (buff_reset) {
+        xqc_conn_buff_1rtt_packet(conn, packet_out);
+    }
+
     xqc_log(conn->log, XQC_LOG_DEBUG, "|stream_id:%ui|stream_state_send:%d|", stream->stream_id, stream->stream_state_send);
     return XQC_OK;
 
@@ -739,7 +759,25 @@ xqc_write_stop_sending_to_packet(xqc_connection_t *conn, xqc_stream_t *stream,
         return XQC_OK;
     }
 
-    packet_out = xqc_write_new_packet(conn, XQC_PTYPE_NUM);
+    xqc_pkt_type_t pkt_type = XQC_PTYPE_SHORT_HEADER;
+    int support_0rtt = xqc_conn_is_ready_to_send_early_data(conn);
+    xqc_bool_t buff_pkt = XQC_FALSE;
+
+    if (!(conn->conn_flag & XQC_CONN_FLAG_CAN_SEND_1RTT)) {
+        if ((conn->conn_type == XQC_CONN_TYPE_CLIENT) 
+            && (conn->conn_state == XQC_CONN_STATE_CLIENT_INITIAL_SENT) 
+            && support_0rtt)
+        {
+            pkt_type = XQC_PTYPE_0RTT;
+            conn->conn_flag |= XQC_CONN_FLAG_HAS_0RTT;
+            stream->stream_flag |= XQC_STREAM_FLAG_HAS_0RTT;
+
+        } else {
+            buff_pkt = XQC_TRUE;
+        }
+    }
+
+    packet_out = xqc_write_new_packet(conn, pkt_type);
     if (packet_out == NULL) {
         xqc_log(conn->log, XQC_LOG_ERROR, "|xqc_write_new_packet error|");
         return -XQC_EWRITE_PKT;
@@ -753,6 +791,10 @@ xqc_write_stop_sending_to_packet(xqc_connection_t *conn, xqc_stream_t *stream,
 
     packet_out->po_used_size += ret;
 
+    if (buff_pkt) {
+        xqc_conn_buff_1rtt_packet(conn, packet_out);
+    }
+
     return XQC_OK;
 
 error:
@@ -969,6 +1011,7 @@ xqc_write_new_token_to_packet(xqc_connection_t *conn)
     }
 
     packet_out->po_used_size += ret;
+    xqc_send_queue_move_to_high_pri(&packet_out->po_list, conn->conn_send_queue);
 
     return XQC_OK;
 
@@ -1134,6 +1177,7 @@ xqc_write_handshake_done_frame_to_packet(xqc_connection_t *conn)
     }
 
     packet_out->po_used_size += n_written;
+    xqc_send_queue_move_to_high_pri(&packet_out->po_list, conn->conn_send_queue);
 
     return XQC_OK;
 }
@@ -1238,6 +1282,7 @@ xqc_write_retire_conn_id_frame_to_packet(xqc_connection_t *conn, uint64_t seq_nu
     }
 
     packet_out->po_used_size += ret;
+    xqc_send_queue_move_to_high_pri(&packet_out->po_list, conn->conn_send_queue);
 
     return XQC_OK;
 }
@@ -1381,7 +1426,8 @@ xqc_write_ack_mp_to_one_packet(xqc_connection_t *conn, xqc_path_ctx_t *path,
     } else {
         conn->conn_flag &= ~XQC_CONN_FLAG_ACK_HAS_GAP;
     }
-    conn->conn_flag &= ~(XQC_CONN_FLAG_SHOULD_ACK_INIT << pns);
+    path->path_flag &= ~(XQC_PATH_FLAG_SHOULD_ACK_INIT << pns);
+    conn->ack_flag &= ~(1 << (pns + path->path_id * XQC_PNS_N));
 
     return XQC_OK;
 
@@ -1497,4 +1543,4 @@ xqc_write_path_standby_or_available_frame_to_packet(xqc_connection_t *conn, xqc_
     error:
     xqc_maybe_recycle_packet_out(packet_out, conn);
     return ret;
-}
\ No newline at end of file
+}
diff --git a/src/transport/xqc_packet_parser.c b/src/transport/xqc_packet_parser.c
index 5ce865809..c51229a5b 100644
--- a/src/transport/xqc_packet_parser.c
+++ b/src/transport/xqc_packet_parser.c
@@ -294,6 +294,11 @@ xqc_packet_parse_short_header(xqc_connection_t *c, xqc_packet_in_t *packet_in)
                 xqc_log(c->log, XQC_LOG_DEBUG, "|update_path_scid|%s->%s|",
                         xqc_scid_str(&path->path_last_scid), 
                         xqc_dcid_str(&path->path_scid));
+                /* clear recv record and reset packet number */
+                path->path_pn_ctl->ctl_largest_acked_ack[XQC_PNS_APP_DATA] = 0;
+                path->path_send_ctl->ctl_largest_received[XQC_PNS_APP_DATA] = XQC_MAX_UINT64_VALUE;
+                xqc_ack_sent_record_reset(&path->path_pn_ctl->ack_sent_record[XQC_PNS_APP_DATA]);
+                xqc_recv_record_destroy(&path->path_pn_ctl->ctl_recv_record[XQC_PNS_APP_DATA]);
             }
         }
 #endif
diff --git a/src/transport/xqc_recv_record.c b/src/transport/xqc_recv_record.c
index 630d98267..4354abcb7 100644
--- a/src/transport/xqc_recv_record.c
+++ b/src/transport/xqc_recv_record.c
@@ -197,6 +197,7 @@ xqc_recv_record_destroy(xqc_recv_record_t *recv_record)
         xqc_free(pnode);
     }
     recv_record->node_count = 0;
+    recv_record->rr_del_from = 0;
 }
 
 /* 把src的链表逐个节点移动到dst */
@@ -247,7 +248,7 @@ xqc_maybe_should_ack(xqc_connection_t *conn, xqc_path_ctx_t *path, xqc_pn_ctl_t
      * Generating Acknowledgements
      */
 
-    if (conn->conn_flag & (XQC_CONN_FLAG_SHOULD_ACK_INIT << pns)) {
+    if (path->path_flag & (XQC_PATH_FLAG_SHOULD_ACK_INIT << pns)) {
         xqc_log(conn->log, XQC_LOG_DEBUG, "|already yes|");
         return;
     }
@@ -269,7 +270,8 @@ xqc_maybe_should_ack(xqc_connection_t *conn, xqc_path_ctx_t *path, xqc_pn_ctl_t
         || (pns <= XQC_PNS_HSK && send_ctl->ctl_ack_eliciting_pkt[pns] >= 1)
         || (out_of_order && send_ctl->ctl_ack_eliciting_pkt[pns] >= 1))
     {
-        conn->conn_flag |= XQC_CONN_FLAG_SHOULD_ACK_INIT << pns;
+        path->path_flag |= XQC_PATH_FLAG_SHOULD_ACK_INIT << pns;
+        conn->ack_flag |= (1 << (pns + path->path_id * XQC_PNS_N));
         
         xqc_timer_unset(&send_ctl->path_timer_manager, XQC_TIMER_ACK_INIT + pns);
 
@@ -305,6 +307,13 @@ xqc_ack_sent_record_init(xqc_ack_sent_record_t *record)
     return XQC_OK;
 }
 
+void 
+xqc_ack_sent_record_reset(xqc_ack_sent_record_t *record)
+{
+    record->last_add_time = 0;
+    xqc_rarray_reinit(record->ack_sent);
+}
+
 void
 xqc_ack_sent_record_destroy(xqc_ack_sent_record_t *record)
 {
diff --git a/src/transport/xqc_recv_record.h b/src/transport/xqc_recv_record.h
index af1091508..fa44e5cfb 100644
--- a/src/transport/xqc_recv_record.h
+++ b/src/transport/xqc_recv_record.h
@@ -67,6 +67,8 @@ void xqc_maybe_should_ack(xqc_connection_t *conn, xqc_path_ctx_t *path, xqc_pn_c
 
 int xqc_ack_sent_record_init(xqc_ack_sent_record_t *record);
 
+void xqc_ack_sent_record_reset(xqc_ack_sent_record_t *record);
+
 void xqc_ack_sent_record_destroy(xqc_ack_sent_record_t *record);
 
 int xqc_ack_sent_record_add(xqc_ack_sent_record_t *record, xqc_packet_out_t *packet_out, xqc_usec_t srtt, xqc_usec_t now);
diff --git a/src/transport/xqc_reinjection.c b/src/transport/xqc_reinjection.c
index ab07de1a2..d9265dc07 100644
--- a/src/transport/xqc_reinjection.c
+++ b/src/transport/xqc_reinjection.c
@@ -126,10 +126,11 @@ xqc_conn_reinject_unack_packets(xqc_connection_t *conn, xqc_reinjection_mode_t m
             }
 
             xqc_log(conn->log, XQC_LOG_DEBUG, "|MP|REINJ|reinject unacked packets|"
-                    "pkt_num:%ui|size:%ud|pkt_type:%s|frame:%s|",
+                    "pkt_num:%ui|size:%ud|pkt_type:%s|frame:%s|mode:%d|",
                     packet_out->po_pkt.pkt_num, packet_out->po_used_size,
                     xqc_pkt_type_2_str(packet_out->po_pkt.pkt_type),
-                    xqc_frame_type_2_str(packet_out->po_frame_types));
+                    xqc_frame_type_2_str(packet_out->po_frame_types),
+                    mode);
         }
     }
 
diff --git a/src/transport/xqc_send_ctl.c b/src/transport/xqc_send_ctl.c
index 64c67f85d..15168d667 100644
--- a/src/transport/xqc_send_ctl.c
+++ b/src/transport/xqc_send_ctl.c
@@ -711,12 +711,14 @@ xqc_send_ctl_on_packet_sent(xqc_send_ctl_t *send_ctl, xqc_pn_ctl_t *pn_ctl, xqc_
 
         if (packet_out->po_flag & XQC_POF_LOST) {
             ++send_ctl->ctl_lost_count;
+            send_ctl->ctl_recent_lost_count[0]++;
             packet_out->po_flag &= ~XQC_POF_LOST;
 
         }
         
         if (packet_out->po_flag & XQC_POF_TLP) {
             ++send_ctl->ctl_tlp_count;
+            send_ctl->ctl_recent_lost_count[0]++;
             packet_out->po_flag &= ~XQC_POF_TLP;
         }
 
@@ -746,6 +748,7 @@ xqc_send_ctl_on_packet_sent(xqc_send_ctl_t *send_ctl, xqc_pn_ctl_t *pn_ctl, xqc_
         }
 
         ++send_ctl->ctl_send_count;
+        send_ctl->ctl_recent_send_count[0]++;
         xqc_stream_path_metrics_on_send(send_ctl->ctl_conn, packet_out);
 
         send_ctl->ctl_last_inflight_pkt_sent_time = now;
@@ -760,6 +763,17 @@ xqc_send_ctl_on_packet_sent(xqc_send_ctl_t *send_ctl, xqc_pn_ctl_t *pn_ctl, xqc_
 
     send_ctl->ctl_conn->conn_last_send_time = now;
 
+    if (!send_ctl->ctl_recent_stats_timestamp) {
+        send_ctl->ctl_recent_stats_timestamp = now;
+    }
+
+    if (now >= send_ctl->ctl_recent_stats_timestamp + (5 * send_ctl->ctl_srtt)) {
+        send_ctl->ctl_recent_stats_timestamp = now;
+        send_ctl->ctl_recent_lost_count[1] = send_ctl->ctl_recent_lost_count[0];
+        send_ctl->ctl_recent_send_count[1] = send_ctl->ctl_recent_send_count[0];
+        send_ctl->ctl_recent_lost_count[0] = 0;
+        send_ctl->ctl_recent_send_count[0] = 0;
+    }
 }
 
 /**
@@ -1783,3 +1797,20 @@ xqc_send_ctl_get_pkt_num_gap(xqc_send_ctl_t *send_ctl, xqc_pkt_num_space_t pns,
 {
     return back - front;
 }
+
+uint64_t 
+xqc_send_ctl_get_est_bw(xqc_send_ctl_t *send_ctl)
+{
+    if (send_ctl->ctl_cong && send_ctl->ctl_cong_callback) {
+        if (send_ctl->ctl_cong_callback->xqc_cong_ctl_get_bandwidth_estimate) {
+            return send_ctl->ctl_cong_callback->xqc_cong_ctl_get_bandwidth_estimate(send_ctl->ctl_cong);
+
+        } else if (send_ctl->ctl_cong_callback->xqc_cong_ctl_get_cwnd) {
+            uint64_t cwnd = send_ctl->ctl_cong_callback->xqc_cong_ctl_get_cwnd(send_ctl->ctl_cong);
+            xqc_usec_t srtt = send_ctl->ctl_srtt ? send_ctl->ctl_srtt : XQC_kInitialRtt * 1000;
+            return (cwnd * 1000000) / srtt;
+        }
+    }
+    
+    return 0;
+}
diff --git a/src/transport/xqc_send_ctl.h b/src/transport/xqc_send_ctl.h
index 7ca18f30b..dcf946e87 100644
--- a/src/transport/xqc_send_ctl.h
+++ b/src/transport/xqc_send_ctl.h
@@ -150,6 +150,11 @@ typedef struct xqc_send_ctl_s {
     xqc_sample_t                sampler;
 
     xqc_send_ctl_info_t         ctl_info;
+
+    unsigned                    ctl_recent_send_count[2];
+    unsigned                    ctl_recent_lost_count[2];
+    xqc_usec_t                  ctl_recent_stats_timestamp;
+
 } xqc_send_ctl_t;
 
 
@@ -254,4 +259,7 @@ xqc_packet_number_t xqc_send_ctl_get_lost_sent_pn(xqc_send_ctl_t *send_ctl, xqc_
 
 xqc_packet_number_t xqc_send_ctl_get_pkt_num_gap(xqc_send_ctl_t *send_ctl, xqc_pkt_num_space_t pns, xqc_packet_number_t front, xqc_packet_number_t back);
 
+/* bytes per second */
+uint64_t xqc_send_ctl_get_est_bw(xqc_send_ctl_t *send_ctl);
+
 #endif /* _XQC_SEND_CTL_H_INCLUDED_ */
diff --git a/src/transport/xqc_timer.c b/src/transport/xqc_timer.c
index cc4e66e1d..3cfea66c2 100644
--- a/src/transport/xqc_timer.c
+++ b/src/transport/xqc_timer.c
@@ -44,7 +44,8 @@ xqc_timer_ack_timeout(xqc_timer_type_t type, xqc_usec_t now, void *user_data)
 
     xqc_connection_t *conn = send_ctl->ctl_conn;
     xqc_pkt_num_space_t pns = type - XQC_TIMER_ACK_INIT;
-    conn->conn_flag |= XQC_CONN_FLAG_SHOULD_ACK_INIT << pns;
+    send_ctl->ctl_path->path_flag |= XQC_PATH_FLAG_SHOULD_ACK_INIT << pns;
+    conn->ack_flag |= (1 << (pns + send_ctl->ctl_path->path_id * XQC_PNS_N));
 
     xqc_log(conn->log, XQC_LOG_DEBUG, "|pns:%d|path:%ui|", pns, send_ctl->ctl_path->path_id);
 }

From 6af83b08fc1214566db4cbe257e35fcfd436b3da Mon Sep 17 00:00:00 2001
From: Yanmei-Liu <miaoji.lym@alibaba-inc.com>
Date: Wed, 15 Nov 2023 19:18:26 +0800
Subject: [PATCH 5/7] [+] add new transport parameter max_concurrent_paths

---
 include/xquic/xquic_typedef.h        |  1 +
 src/transport/xqc_cid.c              |  1 +
 src/transport/xqc_conn.c             |  4 ++++
 src/transport/xqc_conn.h             |  1 +
 src/transport/xqc_transport_params.c | 24 ++++++++++++++++++++++++
 src/transport/xqc_transport_params.h | 10 ++++++++--
 6 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/include/xquic/xquic_typedef.h b/include/xquic/xquic_typedef.h
index d429fb1fa..ab5bc666d 100644
--- a/include/xquic/xquic_typedef.h
+++ b/include/xquic/xquic_typedef.h
@@ -83,6 +83,7 @@ typedef struct xqc_cid_s {
     uint8_t             cid_buf[XQC_MAX_CID_LEN];
     uint64_t            cid_seq_num;
     uint8_t             sr_token[XQC_STATELESS_RESET_TOKENLEN];
+    uint64_t            path_id;    /* preallocate for multi-path */
 } xqc_cid_t;
 
 typedef enum xqc_log_level_s {
diff --git a/src/transport/xqc_cid.c b/src/transport/xqc_cid.c
index 68a9d3ff0..9739ba53e 100644
--- a/src/transport/xqc_cid.c
+++ b/src/transport/xqc_cid.c
@@ -64,6 +64,7 @@ xqc_cid_copy(xqc_cid_t *dst, xqc_cid_t *src)
     xqc_memcpy(dst->cid_buf, src->cid_buf, dst->cid_len);
     dst->cid_seq_num = src->cid_seq_num;
     xqc_memcpy(dst->sr_token, src->sr_token, XQC_STATELESS_RESET_TOKENLEN);
+    dst->path_id = src->path_id;
 }
 
 void
diff --git a/src/transport/xqc_conn.c b/src/transport/xqc_conn.c
index 2ce74cf73..7ee29c200 100644
--- a/src/transport/xqc_conn.c
+++ b/src/transport/xqc_conn.c
@@ -321,6 +321,7 @@ xqc_conn_set_default_settings(xqc_trans_settings_t *settings)
     settings->ack_delay_exponent         = XQC_DEFAULT_ACK_DELAY_EXPONENT;
     settings->max_udp_payload_size       = XQC_DEFAULT_MAX_UDP_PAYLOAD_SIZE;
     settings->active_connection_id_limit = XQC_DEFAULT_ACTIVE_CONNECTION_ID_LIMIT;
+    settings->max_concurrent_paths       = XQC_DEFAULT_MAX_CONCURRENT_PATHS;
 }
 
 static inline void
@@ -4621,6 +4622,7 @@ xqc_conn_set_remote_transport_params(xqc_connection_t *conn,
 
     settings->enable_multipath = params->enable_multipath;
     settings->multipath_version = params->multipath_version;
+    settings->max_concurrent_paths = params->max_concurrent_paths;
     settings->max_datagram_frame_size = params->max_datagram_frame_size;
 
     return XQC_OK;
@@ -4658,6 +4660,7 @@ xqc_conn_get_local_transport_params(xqc_connection_t *conn, xqc_transport_params
     params->no_crypto = settings->no_crypto;
     params->enable_multipath = settings->enable_multipath;
     params->multipath_version = settings->multipath_version;
+    params->max_concurrent_paths = settings->max_concurrent_paths;
     params->max_datagram_frame_size = settings->max_datagram_frame_size;
 
     /* set other transport parameters */
@@ -4958,6 +4961,7 @@ xqc_settings_copy_from_transport_params(xqc_trans_settings_t *dest,
     dest->active_connection_id_limit = src->active_connection_id_limit;
 
     dest->enable_multipath = src->enable_multipath;
+    dest->max_concurrent_paths = src->max_concurrent_paths;
     dest->max_datagram_frame_size = src->max_datagram_frame_size;
 }
 
diff --git a/src/transport/xqc_conn.h b/src/transport/xqc_conn.h
index 86d843987..e8a019d19 100644
--- a/src/transport/xqc_conn.h
+++ b/src/transport/xqc_conn.h
@@ -203,6 +203,7 @@ typedef struct {
     uint64_t                enable_multipath;
     xqc_multipath_version_t multipath_version;
     uint16_t                max_datagram_frame_size;
+    uint64_t                max_concurrent_paths;
 } xqc_trans_settings_t;
 
 
diff --git a/src/transport/xqc_transport_params.c b/src/transport/xqc_transport_params.c
index 944ee5e63..de7063a8f 100644
--- a/src/transport/xqc_transport_params.c
+++ b/src/transport/xqc_transport_params.c
@@ -172,6 +172,12 @@ xqc_transport_params_calc_length(const xqc_transport_params_t *params,
                 xqc_put_varint_len(xqc_put_varint_len(params->enable_multipath)) +
                 xqc_put_varint_len(params->enable_multipath);
         }
+
+        if (params->max_concurrent_paths != XQC_DEFAULT_MAX_CONCURRENT_PATHS) {
+            len += xqc_put_varint_len(XQC_TRANSPORT_PARAM_MAX_CONCURRENT_PATHS) +
+                   xqc_put_varint_len(xqc_put_varint_len(params->max_concurrent_paths)) +
+                   xqc_put_varint_len(params->max_concurrent_paths);
+        }
     }
 
     if (params->max_datagram_frame_size) {
@@ -353,6 +359,11 @@ xqc_encode_transport_params(const xqc_transport_params_t *params,
         } else {
             p = xqc_put_varint_param(p, XQC_TRANSPORT_PARAM_ENABLE_MULTIPATH_04, params->enable_multipath);
         }
+
+        if (params->max_concurrent_paths != XQC_DEFAULT_MAX_CONCURRENT_PATHS) {
+            p = xqc_put_varint_param(p, XQC_TRANSPORT_PARAM_MAX_CONCURRENT_PATHS,
+                                     params->max_concurrent_paths);
+        }
     }
 
     if ((size_t)(p - out) != len) {
@@ -615,6 +626,13 @@ xqc_decode_enable_multipath(xqc_transport_params_t *params, xqc_transport_params
     return XQC_OK;
 }
 
+static xqc_int_t
+xqc_decode_max_concurrent_paths(xqc_transport_params_t *params, xqc_transport_params_type_t exttype,
+    const uint8_t *p, const uint8_t *end, uint64_t param_type, uint64_t param_len)
+{
+    XQC_DECODE_VINT_VALUE(&params->max_concurrent_paths, p, end);
+}
+
 static xqc_int_t
 xqc_decode_max_datagram_frame_size(xqc_transport_params_t *params, xqc_transport_params_type_t exttype,
     const uint8_t *p, const uint8_t *end, uint64_t param_type, uint64_t param_len)
@@ -649,6 +667,7 @@ xqc_trans_param_decode_func xqc_trans_param_decode_func_list[] = {
     xqc_decode_no_crypto,
     xqc_decode_enable_multipath,
     xqc_decode_max_datagram_frame_size,
+    xqc_decode_max_concurrent_paths,
 };
 
 
@@ -688,6 +707,9 @@ xqc_trans_param_get_index(uint64_t param_type)
     case XQC_TRANSPORT_PARAM_MAX_DATAGRAM_FRAME_SIZE:
         return XQC_TRANSPORT_PARAM_PROTOCOL_MAX + 2;
 
+    case XQC_TRANSPORT_PARAM_MAX_CONCURRENT_PATHS:
+        return XQC_TRANSPORT_PARAM_PROTOCOL_MAX + 3;
+
     default:
         break;
     }
@@ -780,6 +802,7 @@ xqc_decode_transport_params(xqc_transport_params_t *params,
 
     params->enable_multipath = 0;
     params->multipath_version = XQC_ERR_MULTIPATH_VERSION;
+    params->max_concurrent_paths = XQC_DEFAULT_MAX_CONCURRENT_PATHS;
 
     while (p < end) {
         ret = xqc_decode_one_transport_param(params, exttype, &p, end);
@@ -910,4 +933,5 @@ xqc_init_transport_params(xqc_transport_params_t *params)
     params->ack_delay_exponent = XQC_DEFAULT_ACK_DELAY_EXPONENT;
     params->max_udp_payload_size = XQC_DEFAULT_MAX_UDP_PAYLOAD_SIZE;
     params->active_connection_id_limit = XQC_DEFAULT_ACTIVE_CONNECTION_ID_LIMIT;
+    params->max_concurrent_paths = XQC_DEFAULT_MAX_CONCURRENT_PATHS;
 }
diff --git a/src/transport/xqc_transport_params.h b/src/transport/xqc_transport_params.h
index 2abbf6897..9daa62235 100644
--- a/src/transport/xqc_transport_params.h
+++ b/src/transport/xqc_transport_params.h
@@ -23,7 +23,8 @@
 /* max buffer length of encoded transport parameter */
 #define XQC_MAX_TRANSPORT_PARAM_BUF_LEN         512
 
-
+/* default value for max_concurrent_paths */
+#define XQC_DEFAULT_MAX_CONCURRENT_PATHS        4
 
 /**
  * @brief transport parameter type
@@ -74,6 +75,9 @@ typedef enum {
     XQC_TRANSPORT_PARAM_ENABLE_MULTIPATH_05                 = 0x0f739bbc1b666d05,
     XQC_TRANSPORT_PARAM_ENABLE_MULTIPATH_06                 = 0x0f739bbc1b666d06,
 
+    /* multipath max concurrent paths */
+    XQC_TRANSPORT_PARAM_MAX_CONCURRENT_PATHS                = 0x0f739bbc1b666df1,
+
     /* upper limit of params defined by xquic */
     XQC_TRANSPORT_PARAM_UNKNOWN,
 } xqc_transport_param_id_t;
@@ -146,7 +150,9 @@ typedef struct {
     uint64_t                enable_multipath;
 
 
-    xqc_multipath_version_t   multipath_version;
+    xqc_multipath_version_t multipath_version;
+
+    uint64_t                max_concurrent_paths;
 
 } xqc_transport_params_t;
 

From 9be5a2fc75f2b607a16dfead81a1b9f57c16c3f3 Mon Sep 17 00:00:00 2001
From: Yanmei-Liu <miaoji.lym@alibaba-inc.com>
Date: Wed, 15 Nov 2023 19:43:22 +0800
Subject: [PATCH 6/7] [+] add option for demo client

---
 demo/demo_client.c    | 15 +++++++++++++--
 include/xquic/xquic.h |  1 +
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/demo/demo_client.c b/demo/demo_client.c
index 1738a5692..989654826 100644
--- a/demo/demo_client.c
+++ b/demo/demo_client.c
@@ -171,6 +171,8 @@ typedef struct xqc_demo_cli_quic_config_s {
 
     uint8_t mp_version;
 
+    uint64_t max_concurrent_paths;
+
     /* support interop test */
     int is_interop_mode;
 
@@ -1629,6 +1631,7 @@ xqc_demo_cli_init_conneciton_settings(xqc_conn_settings_t* settings,
     settings->standby_path_probe_timeout = 1000;
     settings->multipath_version = args->quic_cfg.mp_version;
     settings->mp_ping_on = 1;
+    settings->max_concurrent_paths = args->quic_cfg.max_concurrent_paths;
     settings->is_interop_mode = args->quic_cfg.is_interop_mode;
     if (args->req_cfg.throttled_req != -1) {
         settings->enable_stream_rate_limit = 1;
@@ -1659,6 +1662,7 @@ xqc_demo_cli_init_args(xqc_demo_cli_client_args_t *args)
     args->quic_cfg.keyupdate_pkt_threshold = UINT64_MAX;
     /* default 04 */
     args->quic_cfg.mp_version = XQC_MULTIPATH_04;
+    args->quic_cfg.max_concurrent_paths = UINT64_MAX;
 
     args->req_cfg.throttled_req = -1;
 
@@ -1774,6 +1778,7 @@ xqc_demo_cli_usage(int argc, char *argv[])
         "   -R    Reinjection (1,2,4) \n"
         "   -V    Multipath Version (4,5,6)\n"
         "   -B    Set initial path standby after recvd first application data, and set initial path available after X ms\n"
+        "   -f    Max concurrent paths\n"
         "   -I    Idle interval between requests (ms)\n"
         "   -n    Throttling the {1,2,...}xn-th requests\n"
         "   -e    NAT rebinding on path 0\n"
@@ -1785,7 +1790,7 @@ void
 xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args)
 {
     int ch = 0;
-    while ((ch = getopt(argc, argv, "a:p:c:Ct:S:0m:A:D:l:L:k:K:U:u:dMoi:w:Ps:bZ:NQT:R:V:B:I:n:eE")) != -1) {
+    while ((ch = getopt(argc, argv, "a:p:c:Ct:S:0m:A:D:l:L:k:K:U:u:dMoi:w:Ps:bZ:NQT:R:V:B:I:n:eEf:")) != -1) {
         switch (ch) {
         /* server ip */
         case 'a':
@@ -2015,7 +2020,13 @@ xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args
         case 'E':
             printf("option rebinding path1 after 3s\n");
             args->net_cfg.rebind_p1 = 1;
-            break;     
+            break;
+
+        /* key update packet threshold */
+        case 'f':
+            printf("max concurrent paths: %s\n", optarg);
+            args->quic_cfg.max_concurrent_paths = atoi(optarg);
+            break;
 
         default:
             printf("other option :%c\n", ch);
diff --git a/include/xquic/xquic.h b/include/xquic/xquic.h
index c8581d78b..d3b50b071 100644
--- a/include/xquic/xquic.h
+++ b/include/xquic/xquic.h
@@ -1163,6 +1163,7 @@ typedef struct xqc_conn_settings_s {
      */
     uint64_t                    enable_multipath;
     xqc_multipath_version_t     multipath_version;
+    uint64_t                    max_concurrent_paths;
     uint64_t                    least_available_cid_count;
 
     /*

From 0615dabe143036021a4f674d07e5556114c2ddc3 Mon Sep 17 00:00:00 2001
From: Yanmei-Liu <miaoji.lym@alibaba-inc.com>
Date: Wed, 15 Nov 2023 20:35:53 +0800
Subject: [PATCH 7/7] [+] get min(local, remote) for max_concurrent_paths

---
 demo/demo_client.c       | 2 +-
 src/transport/xqc_conn.c | 3 +++
 src/transport/xqc_conn.h | 4 +++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/demo/demo_client.c b/demo/demo_client.c
index 989654826..bac0de0e0 100644
--- a/demo/demo_client.c
+++ b/demo/demo_client.c
@@ -1662,7 +1662,7 @@ xqc_demo_cli_init_args(xqc_demo_cli_client_args_t *args)
     args->quic_cfg.keyupdate_pkt_threshold = UINT64_MAX;
     /* default 04 */
     args->quic_cfg.mp_version = XQC_MULTIPATH_04;
-    args->quic_cfg.max_concurrent_paths = UINT64_MAX;
+    args->quic_cfg.max_concurrent_paths = 4;
 
     args->req_cfg.throttled_req = -1;
 
diff --git a/src/transport/xqc_conn.c b/src/transport/xqc_conn.c
index 7ee29c200..ee30563de 100644
--- a/src/transport/xqc_conn.c
+++ b/src/transport/xqc_conn.c
@@ -377,6 +377,7 @@ xqc_conn_init_trans_settings(xqc_connection_t *conn)
     ls->enable_multipath = conn->conn_settings.enable_multipath;
     
     ls->multipath_version = conn->conn_settings.multipath_version;
+    ls->max_concurrent_paths = conn->conn_settings.max_concurrent_paths;
 
     ls->max_datagram_frame_size = conn->conn_settings.max_datagram_frame_size;
     ls->disable_active_migration = ls->enable_multipath ? 0 : 1;
@@ -4784,6 +4785,8 @@ xqc_conn_tls_transport_params_cb(const uint8_t *tp, size_t len, void *user_data)
     xqc_log(conn->log, XQC_LOG_DEBUG, "|1RTT_transport_params|max_datagram_frame_size:%ud|",
             conn->remote_settings.max_datagram_frame_size);
 
+    conn->max_concurrent_paths = xqc_min(conn->local_settings.max_concurrent_paths,
+                                         conn->remote_settings.max_concurrent_paths);
 
     /* save no crypto flag */
     if (params.no_crypto == 1) {
diff --git a/src/transport/xqc_conn.h b/src/transport/xqc_conn.h
index e8a019d19..5f7bbaae6 100644
--- a/src/transport/xqc_conn.h
+++ b/src/transport/xqc_conn.h
@@ -357,7 +357,9 @@ struct xqc_connection_s {
     uint32_t                        create_path_count;
     uint32_t                        validated_path_count;
     uint32_t                        active_path_count;
-    
+
+    uint64_t                        max_concurrent_paths;
+
     const
     xqc_scheduler_callback_t       *scheduler_callback;
     void                           *scheduler;