diff --git a/.gitmodules b/.gitmodules index a675f72114..731ab9dc62 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,4 +15,4 @@ [submodule "submodules/xdp-for-windows"] path = submodules/xdp-for-windows url = https://github.com/microsoft/xdp-for-windows.git - branch = release/1.0 + branch = 300d18c43f989ca22ad1be2f2836e2bbec6a2c4f diff --git a/scripts/prepare-machine.ps1 b/scripts/prepare-machine.ps1 index 318648076d..5c79e5133d 100644 --- a/scripts/prepare-machine.ps1 +++ b/scripts/prepare-machine.ps1 @@ -230,9 +230,17 @@ function Install-SigningCertificates { # NB: XDP can be uninstalled via Uninstall-Xdp function Install-Xdp-Driver { if (!$IsWindows) { return } # Windows only + # Install VCRT. Not required for official XDP builds. + Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe" + Start-Process -FilePath "vc_redist.x64.exe" -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow + Write-Host "Downloading XDP msi" $MsiPath = Join-Path $ArtifactsPath "xdp.msi" Invoke-WebRequest -Uri (Get-Content (Join-Path $PSScriptRoot "xdp.json") | ConvertFrom-Json).installer -OutFile $MsiPath + $CertFileName = 'xdp.cer' + Get-AuthenticodeSignature $MsiPath | Select-Object -ExpandProperty SignerCertificate | Export-Certificate -Type CERT -FilePath $CertFileName + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\root' + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\trustedpublisher' Write-Host "Installing XDP driver" msiexec.exe /i $MsiPath /quiet | Out-Null } diff --git a/scripts/quic_callback.ps1 b/scripts/quic_callback.ps1 index 8351591237..378ccf83f3 100644 --- a/scripts/quic_callback.ps1 +++ b/scripts/quic_callback.ps1 @@ -67,10 +67,18 @@ if ($Command.Contains("/home/secnetperf/_work/quic/artifacts/bin/linux/x64_Relea ./artifacts/bin/windows/x64_Release_schannel/secnetperf -exec:$mode -io:$io -stats:$stats } elseif ($Command.Contains("Install_XDP")) { Write-Host "Executing command: Install_XDP" + # Install VCRT. Not required for official XDP builds. + Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe" + Start-Process -FilePath "vc_redist.x64.exe" -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow + Write-Host "(SERVER) Downloading XDP installer" $installerUri = $Command.Split(";")[1] $msiPath = Repo-Path "xdp.msi" Invoke-WebRequest -Uri $installerUri -OutFile $msiPath -UseBasicParsing + $CertFileName = 'xdp.cer' + Get-AuthenticodeSignature $msiPath | Select-Object -ExpandProperty SignerCertificate | Export-Certificate -Type CERT -FilePath $CertFileName + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\root' + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\trustedpublisher' Write-Host "(SERVER) Installing XDP. Msi path: $msiPath" msiexec.exe /i $msiPath /quiet | Out-Host Wait-DriverStarted "xdp" 10000 diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index 0ecd7c94b0..1b54c84b9d 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -135,11 +135,19 @@ function Wait-DriverStarted { # Download and install XDP on both local and remote machines. function Install-XDP { param ($Session, $RemoteDir) + # Install VCRT. Not required for official XDP builds. + Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe" + Start-Process -FilePath "vc_redist.x64.exe" -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow + $installerUri = (Get-Content (Join-Path $PSScriptRoot "xdp.json") | ConvertFrom-Json).installer $msiPath = Repo-Path "artifacts/xdp.msi" Write-Host "Downloading XDP installer" whoami Invoke-WebRequest -Uri $installerUri -OutFile $msiPath -UseBasicParsing + $CertFileName = 'xdp.cer' + Get-AuthenticodeSignature $msiPath | Select-Object -ExpandProperty SignerCertificate | Export-Certificate -Type CERT -FilePath $CertFileName + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\root' + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\trustedpublisher' Write-Host "Installing XDP driver locally" msiexec.exe /i $msiPath /quiet | Out-Null $Size = Get-FileHash $msiPath @@ -157,6 +165,14 @@ function Install-XDP { Copy-Item -ToSession $Session $msiPath -Destination $remoteMsiPath $WaitDriverStartedStr = "${function:Wait-DriverStarted}" Invoke-Command -Session $Session -ScriptBlock { + # Install VCRT. Not required for official XDP builds. + Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vc_redist.x64.exe" -OutFile "vc_redist.x64.exe" + Start-Process -FilePath "vc_redist.x64.exe" -ArgumentList "/install /quiet /norestart" -Wait -NoNewWindow + + $CertFileName = 'xdp.cer' + Get-AuthenticodeSignature $Using:remoteMsiPath | Select-Object -ExpandProperty SignerCertificate | Export-Certificate -Type CERT -FilePath $CertFileName + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\root' + Import-Certificate -FilePath $CertFileName -CertStoreLocation 'cert:\localmachine\trustedpublisher' msiexec.exe /i $Using:remoteMsiPath /quiet | Out-Host $WaitDriverStarted = [scriptblock]::Create($Using:WaitDriverStartedStr) & $WaitDriverStarted xdp 10000 diff --git a/scripts/xdp.json b/scripts/xdp.json index 10371d3402..3de7a2f0e6 100644 --- a/scripts/xdp.json +++ b/scripts/xdp.json @@ -1,3 +1,3 @@ { - "installer": "https://github.com/microsoft/xdp-for-windows/releases/download/v1.0.1/xdp-for-windows.1.0.1.msi" + "installer": "https://github.com/microsoft/xdp-for-windows/releases/download/v1.1.0%2B2886c03f/xdp-for-windows.x64.1.1.0.msi" } diff --git a/src/generated/linux/datapath_raw_xdp_win.c.clog.h b/src/generated/linux/datapath_raw_xdp_win.c.clog.h index b8e8a3d927..a659767e15 100644 --- a/src/generated/linux/datapath_raw_xdp_win.c.clog.h +++ b/src/generated/linux/datapath_raw_xdp_win.c.clog.h @@ -71,26 +71,6 @@ tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInterfaceQueues , arg2, arg3);\ -/*---------------------------------------------------------- -// Decoder Ring for XdpInitialize -// [ xdp][%p] XDP initialized, %u procs -// QuicTraceLogVerbose( - XdpInitialize, - "[ xdp][%p] XDP initialized, %u procs", - Xdp, - Xdp->PartitionCount); -// arg2 = arg2 = Xdp = arg2 -// arg3 = arg3 = Xdp->PartitionCount = arg3 -----------------------------------------------------------*/ -#ifndef _clog_4_ARGS_TRACE_XdpInitialize -#define _clog_4_ARGS_TRACE_XdpInitialize(uniqueId, encoded_arg_string, arg2, arg3)\ -tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInitialize , arg2, arg3);\ - -#endif - - - - /*---------------------------------------------------------- // Decoder Ring for XdpInterfaceInitialize // [ixdp][%p] Initializing interface %u @@ -151,6 +131,26 @@ tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpWorkerStart , arg2, arg3);\ +/*---------------------------------------------------------- +// Decoder Ring for XdpInitialize +// [ xdp][%p] XDP initialized, %u procs +// QuicTraceLogVerbose( + XdpInitialize, + "[ xdp][%p] XDP initialized, %u procs", + Xdp, + Xdp->PartitionCount); +// arg2 = arg2 = Xdp = arg2 +// arg3 = arg3 = Xdp->PartitionCount = arg3 +----------------------------------------------------------*/ +#ifndef _clog_4_ARGS_TRACE_XdpInitialize +#define _clog_4_ARGS_TRACE_XdpInitialize(uniqueId, encoded_arg_string, arg2, arg3)\ +tracepoint(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInitialize , arg2, arg3);\ + +#endif + + + + /*---------------------------------------------------------- // Decoder Ring for XdpRelease // [ xdp][%p] XDP release diff --git a/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h b/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h index 891a5d88f3..c4e0be72f0 100644 --- a/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h +++ b/src/generated/linux/datapath_raw_xdp_win.c.clog.h.lttng.h @@ -51,29 +51,6 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInterfaceQueues, -/*---------------------------------------------------------- -// Decoder Ring for XdpInitialize -// [ xdp][%p] XDP initialized, %u procs -// QuicTraceLogVerbose( - XdpInitialize, - "[ xdp][%p] XDP initialized, %u procs", - Xdp, - Xdp->PartitionCount); -// arg2 = arg2 = Xdp = arg2 -// arg3 = arg3 = Xdp->PartitionCount = arg3 -----------------------------------------------------------*/ -TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInitialize, - TP_ARGS( - const void *, arg2, - unsigned int, arg3), - TP_FIELDS( - ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) - ctf_integer(unsigned int, arg3, arg3) - ) -) - - - /*---------------------------------------------------------- // Decoder Ring for XdpInterfaceInitialize // [ixdp][%p] Initializing interface %u @@ -143,6 +120,29 @@ TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpWorkerStart, +/*---------------------------------------------------------- +// Decoder Ring for XdpInitialize +// [ xdp][%p] XDP initialized, %u procs +// QuicTraceLogVerbose( + XdpInitialize, + "[ xdp][%p] XDP initialized, %u procs", + Xdp, + Xdp->PartitionCount); +// arg2 = arg2 = Xdp = arg2 +// arg3 = arg3 = Xdp->PartitionCount = arg3 +----------------------------------------------------------*/ +TRACEPOINT_EVENT(CLOG_DATAPATH_RAW_XDP_WIN_C, XdpInitialize, + TP_ARGS( + const void *, arg2, + unsigned int, arg3), + TP_FIELDS( + ctf_integer_hex(uint64_t, arg2, (uint64_t)arg2) + ctf_integer(unsigned int, arg3, arg3) + ) +) + + + /*---------------------------------------------------------- // Decoder Ring for XdpRelease // [ xdp][%p] XDP release diff --git a/src/inc/msquic_winuser.h b/src/inc/msquic_winuser.h index adc2f2292e..8157ecd6d1 100644 --- a/src/inc/msquic_winuser.h +++ b/src/inc/msquic_winuser.h @@ -99,6 +99,7 @@ #define QUIC_STATUS_INVALID_STATE E_NOT_VALID_STATE // 0x8007139f #define QUIC_STATUS_NOT_SUPPORTED E_NOINTERFACE // 0x80004002 #define QUIC_STATUS_NOT_FOUND HRESULT_FROM_WIN32(ERROR_NOT_FOUND) // 0x80070490 +#define QUIC_STATUS_FILE_NOT_FOUND HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND) // 0x80070002 #define QUIC_STATUS_BUFFER_TOO_SMALL E_NOT_SUFFICIENT_BUFFER // 0x8007007a #define QUIC_STATUS_HANDSHAKE_FAILURE ERROR_QUIC_HANDSHAKE_FAILURE // 0x80410000 #define QUIC_STATUS_ABORTED E_ABORT // 0x80004004 diff --git a/src/platform/datapath_raw_xdp_win.c b/src/platform/datapath_raw_xdp_win.c index 461d4fcdec..7234b2f508 100644 --- a/src/platform/datapath_raw_xdp_win.c +++ b/src/platform/datapath_raw_xdp_win.c @@ -10,7 +10,10 @@ --*/ #define _CRT_SECURE_NO_WARNINGS 1 // TODO - Remove +#define XDP_API_VERSION 3 +#define XDP_INCLUDE_WINCOMMON +#include #include "datapath_raw_win.h" #include "datapath_raw_xdp.h" #include @@ -42,9 +45,6 @@ typedef struct XDP_DATAPATH { BOOLEAN TxAlwaysPoke; BOOLEAN SkipXsum; BOOLEAN Running; // Signal to stop partitions. - XDP_LOAD_API_CONTEXT XdpApiLoadContext; - const XDP_API_TABLE *XdpApi; - XDP_QEO_SET_FN *XdpQeoSet; XDP_PARTITION Partitions[0]; } XDP_DATAPATH; @@ -159,7 +159,6 @@ CreateNoOpEthernetPacket( QUIC_STATUS CxPlatGetRssQueueProcessors( - _In_ XDP_DATAPATH* Xdp, _In_ uint32_t InterfaceIndex, _Inout_ uint16_t* Count, _Out_writes_to_(*Count, *Count) uint32_t* Queues @@ -171,26 +170,32 @@ CxPlatGetRssQueueProcessors( for (uint16_t i = 0; i < *Count; ++i) { HANDLE TxXsk = NULL; - QUIC_STATUS Status = Xdp->XdpApi->XskCreate(&TxXsk); + QUIC_STATUS Status = XskCreate(&TxXsk); if (QUIC_FAILED(Status)) { return Status; } XSK_UMEM_REG TxUmem = {0}; + UINT32 EnableAffinity = TRUE; TxUmem.Address = &TxPacket; TxUmem.ChunkSize = sizeof(XDP_TX_PACKET); TxUmem.Headroom = FIELD_OFFSET(XDP_TX_PACKET, FrameBuffer); TxUmem.TotalSize = sizeof(XDP_TX_PACKET); - Status = Xdp->XdpApi->XskSetSockopt(TxXsk, XSK_SOCKOPT_UMEM_REG, &TxUmem, sizeof(TxUmem)); + Status = XskSetSockopt(TxXsk, XSK_SOCKOPT_UMEM_REG, &TxUmem, sizeof(TxUmem)); if (QUIC_FAILED(Status)) { CloseHandle(TxXsk); return Status; } - Status = Xdp->XdpApi->XskSetSockopt(TxXsk, XSK_SOCKOPT_TX_RING_SIZE, &TxRingSize, sizeof(TxRingSize)); + Status = XskSetSockopt(TxXsk, XSK_SOCKOPT_TX_RING_SIZE, &TxRingSize, sizeof(TxRingSize)); if (QUIC_FAILED(Status)) { CloseHandle(TxXsk); return Status; } - Status = Xdp->XdpApi->XskSetSockopt(TxXsk, XSK_SOCKOPT_TX_COMPLETION_RING_SIZE, &TxRingSize, sizeof(TxRingSize)); + Status = XskSetSockopt(TxXsk, XSK_SOCKOPT_TX_COMPLETION_RING_SIZE, &TxRingSize, sizeof(TxRingSize)); if (QUIC_FAILED(Status)) { CloseHandle(TxXsk); return Status; } + // + // This is best effort, as the option isn't available on all versions of XDP. + // + Status = XskSetSockopt(TxXsk, XSK_SOCKOPT_TX_PROCESSOR_AFFINITY, &EnableAffinity, sizeof(EnableAffinity)); + uint32_t Flags = XSK_BIND_FLAG_TX; - Status = Xdp->XdpApi->XskBind(TxXsk, InterfaceIndex, i, Flags); + Status = XskBind(TxXsk, InterfaceIndex, i, Flags); if (QUIC_FAILED(Status)) { CloseHandle(TxXsk); if (Status == E_INVALIDARG) { // No more queues. Break out. @@ -205,7 +210,7 @@ CxPlatGetRssQueueProcessors( return Status; } - Status = Xdp->XdpApi->XskActivate(TxXsk, 0); + Status = XskActivate(TxXsk, 0); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -218,7 +223,7 @@ CxPlatGetRssQueueProcessors( XSK_RING_INFO_SET TxRingInfo; uint32_t TxRingInfoSize = sizeof(TxRingInfo); - Status = Xdp->XdpApi->XskGetSockopt(TxXsk, XSK_SOCKOPT_RING_INFO, &TxRingInfo, &TxRingInfoSize); + Status = XskGetSockopt(TxXsk, XSK_SOCKOPT_RING_INFO, &TxRingInfo, &TxRingInfoSize); if (QUIC_FAILED(Status)) { CloseHandle(TxXsk); return Status; } XSK_RING TxRing, TxCompletionRing; @@ -235,7 +240,7 @@ CxPlatGetRssQueueProcessors( XskRingProducerSubmit(&TxRing, 1); XSK_NOTIFY_RESULT_FLAGS OutFlags; - Status = Xdp->XdpApi->XskNotifySocket(TxXsk, XSK_NOTIFY_FLAG_POKE_TX|XSK_NOTIFY_FLAG_WAIT_TX, XDP_MAX_SYNC_WAIT_TIMEOUT_MS, &OutFlags); + Status = XskNotifySocket(TxXsk, XSK_NOTIFY_FLAG_POKE_TX|XSK_NOTIFY_FLAG_WAIT_TX, XDP_MAX_SYNC_WAIT_TIMEOUT_MS, &OutFlags); if (QUIC_FAILED(Status)) { CloseHandle(TxXsk); return Status; } uint32_t CompIndex; @@ -247,7 +252,7 @@ CxPlatGetRssQueueProcessors( PROCESSOR_NUMBER ProcNumber; uint32_t ProcNumberSize = sizeof(PROCESSOR_NUMBER); - Status = Xdp->XdpApi->XskGetSockopt(TxXsk, XSK_SOCKOPT_TX_PROCESSOR_AFFINITY, &ProcNumber, &ProcNumberSize); + Status = XskGetSockopt(TxXsk, XSK_SOCKOPT_TX_PROCESSOR_AFFINITY, &ProcNumber, &ProcNumberSize); if (QUIC_FAILED(Status)) { CloseHandle(TxXsk); return Status; } const CXPLAT_PROCESSOR_GROUP_INFO* Group = &CxPlatProcessorGroupInfo[ProcNumber.Group]; @@ -400,7 +405,7 @@ CxPlatDpRawInterfaceInitialize( goto Error; } - Status = Xdp->XdpApi->XdpInterfaceOpen(Interface->ActualIfIndex, &Interface->XdpHandle); + Status = XdpInterfaceOpen(Interface->ActualIfIndex, &Interface->XdpHandle); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -410,7 +415,7 @@ CxPlatDpRawInterfaceInitialize( goto Error; } - Status = CxPlatGetRssQueueProcessors(Xdp, Interface->ActualIfIndex, &Interface->QueueCount, Processors); + Status = CxPlatGetRssQueueProcessors(Interface->ActualIfIndex, &Interface->QueueCount, Processors); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -479,7 +484,7 @@ CxPlatDpRawInterfaceInitialize( goto Error; } - Status = Xdp->XdpApi->XskCreate(&Queue->RxXsk); + Status = XskCreate(&Queue->RxXsk); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -495,7 +500,7 @@ CxPlatDpRawInterfaceInitialize( RxUmem.Headroom = RxHeadroom; RxUmem.TotalSize = Xdp->RxBufferCount * RxPacketSize; - Status = Xdp->XdpApi->XskSetSockopt(Queue->RxXsk, XSK_SOCKOPT_UMEM_REG, &RxUmem, sizeof(RxUmem)); + Status = XskSetSockopt(Queue->RxXsk, XSK_SOCKOPT_UMEM_REG, &RxUmem, sizeof(RxUmem)); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -506,7 +511,7 @@ CxPlatDpRawInterfaceInitialize( } Status = - Xdp->XdpApi->XskSetSockopt( + XskSetSockopt( Queue->RxXsk, XSK_SOCKOPT_RX_FILL_RING_SIZE, &Xdp->RxRingSize, sizeof(Xdp->RxRingSize)); if (QUIC_FAILED(Status)) { @@ -519,7 +524,7 @@ CxPlatDpRawInterfaceInitialize( } Status = - Xdp->XdpApi->XskSetSockopt( + XskSetSockopt( Queue->RxXsk, XSK_SOCKOPT_RX_RING_SIZE, &Xdp->RxRingSize, sizeof(Xdp->RxRingSize)); if (QUIC_FAILED(Status)) { QuicTraceEvent( @@ -531,7 +536,7 @@ CxPlatDpRawInterfaceInitialize( } uint32_t Flags = XSK_BIND_FLAG_RX; - Status = Xdp->XdpApi->XskBind(Queue->RxXsk, Interface->ActualIfIndex, i, Flags); + Status = XskBind(Queue->RxXsk, Interface->ActualIfIndex, i, Flags); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -541,7 +546,7 @@ CxPlatDpRawInterfaceInitialize( goto Error; } - Status = Xdp->XdpApi->XskActivate(Queue->RxXsk, 0); + Status = XskActivate(Queue->RxXsk, 0); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -553,7 +558,7 @@ CxPlatDpRawInterfaceInitialize( XSK_RING_INFO_SET RxRingInfo; uint32_t RxRingInfoSize = sizeof(RxRingInfo); - Status = Xdp->XdpApi->XskGetSockopt(Queue->RxXsk, XSK_SOCKOPT_RING_INFO, &RxRingInfo, &RxRingInfoSize); + Status = XskGetSockopt(Queue->RxXsk, XSK_SOCKOPT_RING_INFO, &RxRingInfo, &RxRingInfoSize); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -601,7 +606,7 @@ CxPlatDpRawInterfaceInitialize( goto Error; } - Status = Xdp->XdpApi->XskCreate(&Queue->TxXsk); + Status = XskCreate(&Queue->TxXsk); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -617,7 +622,7 @@ CxPlatDpRawInterfaceInitialize( TxUmem.Headroom = FIELD_OFFSET(XDP_TX_PACKET, FrameBuffer); TxUmem.TotalSize = Xdp->TxBufferCount * sizeof(XDP_TX_PACKET); - Status = Xdp->XdpApi->XskSetSockopt(Queue->TxXsk, XSK_SOCKOPT_UMEM_REG, &TxUmem, sizeof(TxUmem)); + Status = XskSetSockopt(Queue->TxXsk, XSK_SOCKOPT_UMEM_REG, &TxUmem, sizeof(TxUmem)); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -628,7 +633,7 @@ CxPlatDpRawInterfaceInitialize( } Status = - Xdp->XdpApi->XskSetSockopt( + XskSetSockopt( Queue->TxXsk, XSK_SOCKOPT_TX_RING_SIZE, &Xdp->TxRingSize, sizeof(Xdp->TxRingSize)); if (QUIC_FAILED(Status)) { QuicTraceEvent( @@ -640,7 +645,7 @@ CxPlatDpRawInterfaceInitialize( } Status = - Xdp->XdpApi->XskSetSockopt( + XskSetSockopt( Queue->TxXsk, XSK_SOCKOPT_TX_COMPLETION_RING_SIZE, &Xdp->TxRingSize, sizeof(Xdp->TxRingSize)); if (QUIC_FAILED(Status)) { @@ -653,7 +658,7 @@ CxPlatDpRawInterfaceInitialize( } Flags = XSK_BIND_FLAG_TX; // TODO: support native/generic forced flags. - Status = Xdp->XdpApi->XskBind(Queue->TxXsk, Interface->ActualIfIndex, i, Flags); + Status = XskBind(Queue->TxXsk, Interface->ActualIfIndex, i, Flags); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -663,7 +668,7 @@ CxPlatDpRawInterfaceInitialize( goto Error; } - Status = Xdp->XdpApi->XskActivate(Queue->TxXsk, 0); + Status = XskActivate(Queue->TxXsk, 0); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -675,7 +680,7 @@ CxPlatDpRawInterfaceInitialize( XSK_RING_INFO_SET TxRingInfo; uint32_t TxRingInfoSize = sizeof(TxRingInfo); - Status = Xdp->XdpApi->XskGetSockopt(Queue->TxXsk, XSK_SOCKOPT_RING_INFO, &TxRingInfo, &TxRingInfoSize); + Status = XskGetSockopt(Queue->TxXsk, XSK_SOCKOPT_RING_INFO, &TxRingInfo, &TxRingInfoSize); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -765,7 +770,7 @@ CxPlatDpRawInterfaceUpdateRules( HANDLE NewRxProgram; QUIC_STATUS Status = - Interface->Xdp->XdpApi->XdpCreateProgram( + XdpCreateProgram( Interface->ActualIfIndex, &RxHook, i, @@ -937,19 +942,13 @@ CxPlatDpRawInitialize( { XDP_DATAPATH* Xdp = (XDP_DATAPATH*)Datapath; PMIB_IF_TABLE2 pIfTable = NULL; - QUIC_STATUS Status; + QUIC_STATUS Status = QUIC_STATUS_SUCCESS; if (WorkerPool == NULL) { return QUIC_STATUS_INVALID_PARAMETER; } CxPlatListInitializeHead(&Xdp->Interfaces); - if (QUIC_FAILED(XdpLoadApi(XDP_API_VERSION_1, &Xdp->XdpApiLoadContext, &Xdp->XdpApi))) { - Status = QUIC_STATUS_NOT_SUPPORTED; - goto Error; - } - - Xdp->XdpQeoSet = (XDP_QEO_SET_FN *)Xdp->XdpApi->XdpGetRoutine(XDP_QEO_SET_FN_NAME); CxPlatXdpReadConfig(Xdp); Xdp->PollingIdleTimeoutUs = Config ? Config->PollingIdleTimeoutUs : 0; @@ -966,12 +965,6 @@ CxPlatDpRawInitialize( } } - QuicTraceLogVerbose( - XdpInitialize, - "[ xdp][%p] XDP initialized, %u procs", - Xdp, - Xdp->PartitionCount); - if (GetIfTable2(&pIfTable) != NO_ERROR) { Status = QUIC_STATUS_INTERNAL_ERROR; goto Error; @@ -1065,7 +1058,13 @@ CxPlatDpRawInitialize( Status = CxPlatDpRawInterfaceInitialize( Xdp, Interface, ClientRecvContextLength); - if (QUIC_FAILED(Status)) { + if (Status == QUIC_STATUS_FILE_NOT_FOUND && CxPlatListIsEmpty(&Xdp->Interfaces)) { + // + // FILE_NOT_FOUND for the first interface means that XDP is not available on this system. + // + Status = QUIC_STATUS_NOT_SUPPORTED; + break; + } else if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, "[ lib] ERROR, %u, %s.", @@ -1088,11 +1087,18 @@ CxPlatDpRawInitialize( } if (CxPlatListIsEmpty(&Xdp->Interfaces)) { - QuicTraceEvent( - LibraryError, - "[ lib] ERROR, %s.", - "no XDP capable interface"); - Status = QUIC_STATUS_NOT_FOUND; + if (Status == QUIC_STATUS_NOT_SUPPORTED) { + QuicTraceEvent( + LibraryError, + "[ lib] ERROR, %s.", + "XDP is not supported on this system"); + } else { + QuicTraceEvent( + LibraryError, + "[ lib] ERROR, %s.", + "no XDP capable interface"); + Status = QUIC_STATUS_NOT_FOUND; + } goto Error; } @@ -1150,6 +1156,12 @@ CxPlatDpRawInitialize( } Status = QUIC_STATUS_SUCCESS; + QuicTraceLogVerbose( + XdpInitialize, + "[ xdp][%p] XDP initialized, %u procs", + Xdp, + Xdp->PartitionCount); + Error: if (pIfTable != NULL) { FreeMibTable(pIfTable); @@ -1162,10 +1174,6 @@ CxPlatDpRawInitialize( CxPlatDpRawInterfaceUninitialize(Interface); CxPlatFree(Interface, IF_TAG); } - - if (Xdp->XdpApi) { - XdpUnloadApi(Xdp->XdpApiLoadContext, Xdp->XdpApi); - } } return Status; @@ -1192,7 +1200,6 @@ CxPlatDpRawRelease( CxPlatDpRawInterfaceUninitialize(Interface); CxPlatFree(Interface, IF_TAG); } - XdpUnloadApi(Xdp->XdpApiLoadContext, Xdp->XdpApi); CxPlatDataPathUninitializeComplete((CXPLAT_DATAPATH_RAW*)Xdp); } } @@ -1280,15 +1287,11 @@ RawSocketUpdateQeo( BOOLEAN AtLeastOneSucceeded = FALSE; for (CXPLAT_LIST_ENTRY* Entry = Xdp->Interfaces.Flink; Entry != &Xdp->Interfaces; Entry = Entry->Flink) { - if (Xdp->XdpQeoSet != NULL) { - Status = - Xdp->XdpQeoSet( - CONTAINING_RECORD(Entry, XDP_INTERFACE, Link)->XdpHandle, - Connections, - sizeof(Connections)); - } else { - Status = E_NOINTERFACE; - } + Status = + XdpQeoSet( + CONTAINING_RECORD(Entry, XDP_INTERFACE, Link)->XdpHandle, + Connections, + sizeof(Connections)); if (QUIC_FAILED(Status)) { QuicTraceEvent( LibraryErrorStatus, @@ -1545,7 +1548,7 @@ CxPlatXdpRx( XSK_ERROR ErrorStatus; QUIC_STATUS XskStatus; uint32_t ErrorSize = sizeof(ErrorStatus); - XskStatus = Xdp->XdpApi->XskGetSockopt(Queue->RxXsk, XSK_SOCKOPT_RX_ERROR, &ErrorStatus, &ErrorSize); + XskStatus = XskGetSockopt(Queue->RxXsk, XSK_SOCKOPT_RX_ERROR, &ErrorStatus, &ErrorSize); QuicTraceEvent( LibraryErrorStatus, "[ lib] ERROR, %u, %s.", @@ -1704,7 +1707,7 @@ CxPlatXdpTx( MemoryBarrier(); if (Xdp->TxAlwaysPoke || XskRingProducerNeedPoke(&Queue->TxRing)) { XSK_NOTIFY_RESULT_FLAGS OutFlags; - QUIC_STATUS Status = Xdp->XdpApi->XskNotifySocket(Queue->TxXsk, XSK_NOTIFY_FLAG_POKE_TX, 0, &OutFlags); + QUIC_STATUS Status = XskNotifySocket(Queue->TxXsk, XSK_NOTIFY_FLAG_POKE_TX, 0, &OutFlags); CXPLAT_DBG_ASSERT(QUIC_SUCCEEDED(Status)); UNREFERENCED_PARAMETER(Status); } @@ -1714,7 +1717,7 @@ CxPlatXdpTx( XSK_ERROR ErrorStatus; QUIC_STATUS XskStatus; uint32_t ErrorSize = sizeof(ErrorStatus); - XskStatus = Xdp->XdpApi->XskGetSockopt(Queue->TxXsk, XSK_SOCKOPT_TX_ERROR, &ErrorStatus, &ErrorSize); + XskStatus = XskGetSockopt(Queue->TxXsk, XSK_SOCKOPT_TX_ERROR, &ErrorStatus, &ErrorSize); QuicTraceEvent( LibraryErrorStatus, "[ lib] ERROR, %u, %s.", @@ -1783,7 +1786,7 @@ CxPlatXdpExecute( &Queue->RxIoSqe.DatapathSqe.Sqe.Overlapped, sizeof(Queue->RxIoSqe.DatapathSqe.Sqe.Overlapped)); HRESULT hr = - Xdp->XdpApi->XskNotifyAsync( + XskNotifyAsync( Queue->RxXsk, XSK_NOTIFY_FLAG_WAIT_RX, &Queue->RxIoSqe.DatapathSqe.Sqe.Overlapped); if (hr == HRESULT_FROM_WIN32(ERROR_IO_PENDING)) { @@ -1807,7 +1810,7 @@ CxPlatXdpExecute( &Queue->TxIoSqe.DatapathSqe.Sqe.Overlapped, sizeof(Queue->TxIoSqe.DatapathSqe.Sqe.Overlapped)); HRESULT hr = - Xdp->XdpApi->XskNotifyAsync( + XskNotifyAsync( Queue->TxXsk, XSK_NOTIFY_FLAG_WAIT_TX, &Queue->TxIoSqe.DatapathSqe.Sqe.Overlapped); if (hr == HRESULT_FROM_WIN32(ERROR_IO_PENDING)) { diff --git a/submodules/xdp-for-windows b/submodules/xdp-for-windows index 4727f89d80..300d18c43f 160000 --- a/submodules/xdp-for-windows +++ b/submodules/xdp-for-windows @@ -1 +1 @@ -Subproject commit 4727f89d807e2455a236136a66f16d2bf4a66e50 +Subproject commit 300d18c43f989ca22ad1be2f2836e2bbec6a2c4f