[+] path_id first shot (please don't merge)

demo/demo_client.c

@@ -171,6 +171,8 @@ typedef struct xqc_demo_cli_quic_config_s {
 
     uint8_t mp_version;
 
+    uint64_t max_concurrent_paths;
+
     /* support interop test */
     int is_interop_mode;
 
@@ -1629,6 +1631,7 @@ xqc_demo_cli_init_conneciton_settings(xqc_conn_settings_t* settings,
     settings->standby_path_probe_timeout = 1000;
     settings->multipath_version = args->quic_cfg.mp_version;
     settings->mp_ping_on = 1;
+    settings->max_concurrent_paths = args->quic_cfg.max_concurrent_paths;
     settings->is_interop_mode = args->quic_cfg.is_interop_mode;
     if (args->req_cfg.throttled_req != -1) {
         settings->enable_stream_rate_limit = 1;
@@ -1659,6 +1662,7 @@ xqc_demo_cli_init_args(xqc_demo_cli_client_args_t *args)
     args->quic_cfg.keyupdate_pkt_threshold = UINT64_MAX;
     /* default 04 */
     args->quic_cfg.mp_version = XQC_MULTIPATH_04;
+    args->quic_cfg.max_concurrent_paths = 4;
 
     args->req_cfg.throttled_req = -1;
 
@@ -1774,6 +1778,7 @@ xqc_demo_cli_usage(int argc, char *argv[])
         "   -R    Reinjection (1,2,4) \n"
         "   -V    Multipath Version (4,5,6)\n"
         "   -B    Set initial path standby after recvd first application data, and set initial path available after X ms\n"
+        "   -f    Max concurrent paths\n"
         "   -I    Idle interval between requests (ms)\n"
         "   -n    Throttling the {1,2,...}xn-th requests\n"
         "   -e    NAT rebinding on path 0\n"
@@ -1785,7 +1790,7 @@ void
 xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args)
 {
     int ch = 0;
-    while ((ch = getopt(argc, argv, "a:p:c:Ct:S:0m:A:D:l:L:k:K:U:u:dMoi:w:Ps:bZ:NQT:R:V:B:I:n:eE")) != -1) {
+    while ((ch = getopt(argc, argv, "a:p:c:Ct:S:0m:A:D:l:L:k:K:U:u:dMoi:w:Ps:bZ:NQT:R:V:B:I:n:eEf:")) != -1) {
         switch (ch) {
         /* server ip */
         case 'a':
@@ -1926,7 +1931,7 @@ xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args
             printf("option multipath on\n");
             args->net_cfg.multipath = 1;
             break;
-        
+
          case 'o':
             printf("set interop mode\n");
             args->quic_cfg.is_interop_mode = 1;
@@ -2015,7 +2020,13 @@ xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args
         case 'E':
             printf("option rebinding path1 after 3s\n");
             args->net_cfg.rebind_p1 = 1;
-            break;     
+            break;
+
+        /* key update packet threshold */
+        case 'f':
+            printf("max concurrent paths: %s\n", optarg);
+            args->quic_cfg.max_concurrent_paths = atoi(optarg);
+            break;
 
         default:
             printf("other option :%c\n", ch);

include/xquic/xqc_configure.h

@@ -2,9 +2,9 @@
 #define xquic_VERSION_MAJOR 0
 #define xquic_VERSION_MINOR 1
 
-#define XQC_ENABLE_BBR2
-#define XQC_ENABLE_RENO
-#define XQC_ENABLE_COPA
-#define XQC_ENABLE_UNLIMITED
-#define XQC_ENABLE_MP_INTEROP
+/* #undef XQC_ENABLE_BBR2 */
+/* #undef XQC_ENABLE_RENO */
+/* #undef XQC_ENABLE_COPA */
+/* #undef XQC_ENABLE_UNLIMITED */
+/* #undef XQC_ENABLE_MP_INTEROP */
 /* #undef XQC_NO_PID_PACKET_PROCESS */

include/xquic/xquic.h

@@ -838,20 +838,12 @@ typedef struct xqc_cc_params_s {
 } xqc_cc_params_t;
 
 typedef struct xqc_scheduler_params_u {
-    uint8_t customize_on;
-    union {
-        struct {
-            double lambda_max;
-            double lambda_min;
-            double lambda_init;
-            double lambda_inc;
-            double lambda_dec;
-        } blest_params;
-
-        struct {
-            double beta;
-        } ecf_params;
-    };
+    uint64_t    rtt_us_thr_high;
+    uint64_t    rtt_us_thr_low;
+    uint64_t    bw_Bps_thr;
+    double      loss_percent_thr_high;
+    double      loss_percent_thr_low;
+    uint32_t    pto_cnt_thr;
 } xqc_scheduler_params_t;
 
 typedef struct xqc_congestion_control_callback_s {
@@ -1171,6 +1163,7 @@ typedef struct xqc_conn_settings_s {
      */
     uint64_t                    enable_multipath;
     xqc_multipath_version_t     multipath_version;
+    uint64_t                    max_concurrent_paths;
     uint64_t                    least_available_cid_count;
 
     /*
@@ -1330,6 +1323,9 @@ typedef struct xqc_conn_stats_s {
      */
     int                 mp_state;
 
+    int                 total_rebind_count;
+    int                 total_rebind_valid;
+
     xqc_path_metrics_t  paths_info[XQC_MAX_PATHS_COUNT];
     char                conn_info[XQC_CONN_INFO_LEN];
 

include/xquic/xquic_typedef.h

@@ -83,6 +83,7 @@ typedef struct xqc_cid_s {
     uint8_t             cid_buf[XQC_MAX_CID_LEN];
     uint64_t            cid_seq_num;
     uint8_t             sr_token[XQC_STATELESS_RESET_TOKENLEN];
+    uint64_t            path_id;    /* preallocate for multi-path */
 } xqc_cid_t;
 
 typedef enum xqc_log_level_s {

scripts/case_test.sh

@@ -3460,15 +3460,12 @@ clear_log
 echo -e "h3_ext_finish_bytestream_during_transmission...\c"
 ./test_client -l d -T 2 -s 102400 -U 1 -Q 65535 -E -x 304 > stdlog
 cli_res1=`grep ">>>>>>>> pass:1" stdlog | wc -l`
-cli_res2=`grep "\[dgram\]|echo_check|same_content:yes|" stdlog`
-cli_res3=(`grep "\[h3-dgram\]|recv_dgram_bytes:" stdlog | egrep -o ':[0-9]+' | egrep -o '[0-9]+'`)
 cli_res4=(`grep "\[bytestream\]|bytes_sent:" stdlog | egrep -o ':[0-9]+' | egrep -o '[0-9]+'`)
 cli_res5=`grep "\[bytestream\]|same_content:yes|" stdlog | wc -l`
 cli_res6=`grep "send pure fin" clog`
 errlog=`grep_err_log | grep -v "send data after fin sent"`
-if [ "$cli_res1" == "1" ] && [ -n "$cli_res2" ] \
-    && [ ${cli_res3[0]} -ge 102400 ] && [ ${cli_res3[1]} -ge 102400 ] \
-    && [ ${cli_res4[0]} -ge 102400 ] && [ ${cli_res4[1]} -ge 102400 ] \
+if [ "$cli_res1" == "1" ] \
+    && [ ${cli_res4[0]} -eq 102400 ] && [ ${cli_res4[1]} -eq 102400 ] \
     && [ "$cli_res5" == "1" ] && [ -n "$cli_res6" ] && [ -z "$errlog" ]; then
     echo ">>>>>>>> pass:1"
     case_print_result "h3_ext_finish_bytestream_during_transmission" "pass"

scripts/xquic.lds

@@ -116,8 +116,8 @@ XQUIC_VERS_1.0 {
         xqc_conn_get_conn_settings_template;
         xqc_conn_get_lastest_rtt;
         xqc_log_enable;
-	xqc_h3_request_update_settings;
-  	xqc_stream_update_settings;
+        xqc_h3_request_update_settings;
+        xqc_stream_update_settings;
     local:
         *;
 };

src/common/utils/ringarray/xqc_ring_array.c

@@ -239,3 +239,11 @@ xqc_rarray_resize(xqc_rarray_t *ra, uint64_t cap)
 
     return XQC_OK;
 }
+
+void 
+xqc_rarray_reinit(xqc_rarray_t *ra)
+{
+    xqc_memzero(ra->buf, ra->cap * ra->esize);
+    ra->count = 0;
+    ra->offset = 0;
+}

src/common/utils/ringarray/xqc_ring_array.h

@@ -84,4 +84,6 @@ xqc_int_t xqc_rarray_pop_from(xqc_rarray_t *ra, uint64_t idx);
  */
 xqc_int_t xqc_rarray_resize(xqc_rarray_t *ra, uint64_t cap);
 
+void xqc_rarray_reinit(xqc_rarray_t *ra);
+
 #endif

src/http3/xqc_h3_request.c

@@ -177,7 +177,7 @@ xqc_stream_info_print(xqc_h3_stream_t *h3_stream, xqc_request_stats_t *stats)
     size_t cursor = 0, ret = 0;
     int i;
     int flag = 0;
-    uint32_t mp_settings = 0;
+    char mp_settings[XQC_MP_SETTINGS_STR_LEN] = {0};
 
     if (h3c->conn->handshake_complete_time > 0) {
         flag = 1;
@@ -187,32 +187,11 @@ xqc_stream_info_print(xqc_h3_stream_t *h3_stream, xqc_request_stats_t *stats)
         flag |= 1 << 1;
     }
 
-    if (h3c->conn->enable_multipath) {
-        mp_settings |= 1;
-    }
-
-    if (h3c->conn->local_settings.enable_multipath) {
-        mp_settings |= (1 << 1);
-    }
-
-    if (h3c->conn->remote_settings.enable_multipath) {
-        mp_settings |= (1 << 2);
-    }
-
-    if (h3c->conn->conn_settings.multipath_version == XQC_MULTIPATH_05) {
-        mp_settings |= (1 << 3);
-    }
-
-    if (h3c->conn->local_settings.multipath_version == XQC_MULTIPATH_05) {
-        mp_settings |= (1 << 4);
-    }
-
-    if (h3c->conn->remote_settings.multipath_version == XQC_MULTIPATH_05) {
-        mp_settings |= (1 << 5);
-    }
+    xqc_conn_encode_mp_settings(h3c->conn, mp_settings, XQC_MP_SETTINGS_STR_LEN);
 
-    ret = snprintf(buff, buff_size, "(%d,%"PRIu64",%d)#", 
-                   flag, h3_stream->recv_rate_limit, mp_settings);
+    ret = snprintf(buff, buff_size, "(%d,%"PRIu64",%s,%"PRIu64",%"PRIu64")#", 
+                   flag, h3_stream->recv_rate_limit, mp_settings,
+                   h3_stream->send_offset, h3_stream->recv_offset);
 
     cursor += ret;
 

src/http3/xqc_h3_stream.c

@@ -1908,6 +1908,8 @@ xqc_h3_stream_close_notify(xqc_stream_t *stream, void *user_data)
         h3s->h3r->stream_fin_send_time = h3s->stream->stream_stats.local_fin_snd_time;
         h3s->h3r->stream_fin_ack_time = h3s->stream->stream_stats.first_fin_ack_time;
         h3s->h3r->stream_close_msg = h3s->stream->stream_close_msg;
+        h3s->send_offset = h3s->stream->stream_send_offset;
+        h3s->recv_offset = h3s->stream->stream_data_in.merged_offset_end;
     }
 
     if (h3s->h3_ext_bs && h3s->type == XQC_H3_STREAM_TYPE_BYTESTEAM) {

src/http3/xqc_h3_stream.h

@@ -148,6 +148,8 @@ typedef struct xqc_h3_stream_s {
     uint32_t                        ref_cnt;
 
     uint64_t                        recv_rate_limit;
+    uint64_t                        send_offset;
+    uint64_t                        recv_offset;
 
 } xqc_h3_stream_t;
 

src/transport/reinjection_control/xqc_reinj_deadline.c

@@ -42,10 +42,37 @@ xqc_deadline_reinj_ctl_init(void *reinj_ctl, xqc_connection_t *conn)
     rctl->conn = conn;
 }
 
+static inline xqc_bool_t
+xqc_deadline_reinj_check_packet(xqc_packet_out_t *po)
+{
+    if (((po->po_frame_types & XQC_FRAME_BIT_STREAM) 
+         || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA)
+         || (po->po_frame_types & XQC_FRAME_BIT_RESET_STREAM)
+         || (po->po_frame_types & XQC_FRAME_BIT_STOP_SENDING)
+         || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAMS)
+         || (po->po_frame_types & XQC_FRAME_BIT_MAX_DATA)
+         || (po->po_frame_types & XQC_FRAME_BIT_DATA_BLOCKED)
+         || (po->po_frame_types & XQC_FRAME_BIT_STREAM_DATA_BLOCKED)
+         || (po->po_frame_types & XQC_FRAME_BIT_STREAMS_BLOCKED)
+         || (po->po_frame_types & XQC_FRAME_BIT_CONNECTION_CLOSE))
+        && !(po->po_flag & XQC_POF_NOT_REINJECT)
+        && !(XQC_MP_PKT_REINJECTED(po))
+        && (po->po_flag & XQC_POF_IN_FLIGHT)) 
+    {   
+        return XQC_TRUE;
+    }
+
+    return XQC_FALSE;
+}
+
 static xqc_bool_t
 xqc_deadline_reinj_can_reinject_before_sched(xqc_deadline_reinj_ctl_t *rctl, 
     xqc_packet_out_t *po)
 {
+    if (!xqc_deadline_reinj_check_packet(po)) {
+        return XQC_FALSE;
+    }
+
     xqc_connection_t *conn = rctl->conn;
     xqc_usec_t now = xqc_monotonic_timestamp();
     xqc_usec_t min_srtt = xqc_conn_get_min_srtt(conn, 0);
@@ -56,22 +83,47 @@ xqc_deadline_reinj_can_reinject_before_sched(xqc_deadline_reinj_ctl_t *rctl,
     uint64_t lower_bound = conn->conn_settings.reinj_deadline_lower_bound;
     double   deadline    = xqc_max(xqc_min(flexible, (double)hard), (double)lower_bound);
 
-    xqc_log(conn->log, XQC_LOG_DEBUG, "|deadline:%f|factor:%.4f|min_srtt:%ui|flexible:%f|hard:%ui|lower_bound:%ui|now:%ui|sent_time:%ui|frame:%s|",
-                                      deadline, factor, min_srtt, flexible, hard, lower_bound, now, po->po_sent_time, xqc_frame_type_2_str(po->po_frame_types));
+    xqc_log(conn->log, XQC_LOG_DEBUG, 
+            "|deadline:%f|factor:%.4f|min_srtt:%ui|flexible:%f|hard:%ui|"
+            "lower_bound:%ui|now:%ui|sent_time:%ui|frame:%s|",
+            deadline, factor, min_srtt, flexible, hard, 
+            lower_bound, now, po->po_sent_time, 
+            xqc_frame_type_2_str(po->po_frame_types));
 
-    if (((po->po_frame_types & XQC_FRAME_BIT_STREAM) 
-         || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA))
-        && !(po->po_flag & XQC_POF_NOT_REINJECT)
-        && !(XQC_MP_PKT_REINJECTED(po))
-        && (po->po_flag & XQC_POF_IN_FLIGHT)
-        && ((double)(now - po->po_sent_time) >= deadline)) 
-    {   
+    if ((double)(now - po->po_sent_time) >= deadline) {   
         return XQC_TRUE;
     }
 
     return XQC_FALSE;
 }
 
+
+static xqc_bool_t
+xqc_deadline_reinj_can_reinject_after_send(xqc_deadline_reinj_ctl_t *rctl, 
+    xqc_packet_out_t *po)
+{
+    if (!xqc_deadline_reinj_check_packet(po)) {
+        return XQC_FALSE;
+    }
+
+    xqc_connection_t *conn = rctl->conn;
+    xqc_path_ctx_t *path = NULL;
+    xqc_path_perf_class_t path_class;
+
+    path = xqc_conn_find_path_by_path_id(conn, po->po_path_id);
+
+    if (path) {
+        path_class = xqc_path_get_perf_class(path);
+        if (path_class == XQC_PATH_CLASS_STANDBY_LOW 
+            || path_class == XQC_PATH_CLASS_AVAILABLE_LOW)
+        {
+            return XQC_TRUE;
+        }
+    }
+
+    return XQC_FALSE;
+}
+
 static xqc_bool_t
 xqc_deadline_reinj_can_reinject(void *ctl, 
     xqc_packet_out_t *po, xqc_reinjection_mode_t mode)
@@ -83,6 +135,9 @@ xqc_deadline_reinj_can_reinject(void *ctl,
     case XQC_REINJ_UNACK_BEFORE_SCHED:
         can_reinject = xqc_deadline_reinj_can_reinject_before_sched(rctl, po);
         break;
+    case XQC_REINJ_UNACK_AFTER_SEND:
+        can_reinject = xqc_deadline_reinj_can_reinject_after_send(rctl, po);
+        break;
     default:
         can_reinject = XQC_FALSE;
         break;

src/transport/reinjection_control/xqc_reinj_default.c

@@ -50,7 +50,15 @@ xqc_default_reinj_can_reinject_after_sched(xqc_default_reinj_ctl_t *rctl,
 
     if (xqc_list_empty(&conn->conn_send_queue->sndq_send_packets)
         && ((po->po_frame_types & XQC_FRAME_BIT_STREAM) 
-            || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA))
+            || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA)
+            || (po->po_frame_types & XQC_FRAME_BIT_RESET_STREAM)
+            || (po->po_frame_types & XQC_FRAME_BIT_STOP_SENDING)
+            || (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAMS)
+            || (po->po_frame_types & XQC_FRAME_BIT_MAX_DATA)
+            || (po->po_frame_types & XQC_FRAME_BIT_DATA_BLOCKED)
+            || (po->po_frame_types & XQC_FRAME_BIT_STREAM_DATA_BLOCKED)
+            || (po->po_frame_types & XQC_FRAME_BIT_STREAMS_BLOCKED)
+            || (po->po_frame_types & XQC_FRAME_BIT_CONNECTION_CLOSE))
         && !(po->po_flag & XQC_POF_NOT_REINJECT)
         && !(XQC_MP_PKT_REINJECTED(po))
         && (po->po_flag & XQC_POF_IN_FLIGHT)) 

src/transport/reinjection_control/xqc_reinj_dgram.c

@@ -48,7 +48,7 @@ xqc_dgram_reinj_can_reinject_after_send(xqc_dgram_reinj_ctl_t *rctl,
 {
     xqc_connection_t *conn = rctl->conn;
 
-    if ((po->po_frame_types & XQC_FRAME_BIT_DATAGRAM)
+    if ((po->po_frame_types & (XQC_FRAME_BIT_DATAGRAM | XQC_FRAME_BIT_CONNECTION_CLOSE))
         && !(po->po_flag & XQC_POF_NOT_REINJECT)
         && !(XQC_MP_PKT_REINJECTED(po))
         && (po->po_flag & XQC_POF_IN_FLIGHT))