asm_amd64.s

// Code generated by command: go run avo.go -out ../../asm_amd64.s -stubs ../../stubs_amd64.go -pkg vectorcmp --dispatcher-amd64 ../../dispatch_amd64.go --dispatcher-other ../../dispatch_other.go --generate-test ../../generated_test.go. DO NOT EDIT.

//go:build !purego

#include "textflag.h"

DATA constants<>+0(SB)/4, $0x55555555
DATA constants<>+4(SB)/4, $0x11111111
DATA constants<>+8(SB)/4, $0x01010101
GLOBL constants<>(SB), RODATA|NOPTR, $12

DATA const_zeroes<>+0(SB)/4, $0x00000000
DATA const_zeroes<>+4(SB)/4, $0x00000000
DATA const_zeroes<>+8(SB)/4, $0x00000000
DATA const_zeroes<>+12(SB)/4, $0x00000000
GLOBL const_zeroes<>(SB), RODATA|NOPTR, $16

DATA const_onezero<>+0(SB)/4, $0x01000100
DATA const_onezero<>+4(SB)/4, $0x01000100
DATA const_onezero<>+8(SB)/4, $0x01000100
DATA const_onezero<>+12(SB)/4, $0x01000100
GLOBL const_onezero<>(SB), RODATA|NOPTR, $16

DATA const_three_through_zero<>+0(SB)/4, $0x03020100
DATA const_three_through_zero<>+4(SB)/4, $0x03020100
DATA const_three_through_zero<>+8(SB)/4, $0x03020100
DATA const_three_through_zero<>+12(SB)/4, $0x03020100
GLOBL const_three_through_zero<>(SB), RODATA|NOPTR, $16

DATA const_seven_through_zero<>+0(SB)/8, $0x0706050403020100
DATA const_seven_through_zero<>+8(SB)/8, $0x0706050403020100
GLOBL const_seven_through_zero<>(SB), RODATA|NOPTR, $16

// func asmAVX2EqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2EqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQB Y1, Y0, Y1
	VPCMPEQB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQB X1, X0, X1
	VPCMPEQB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2GreaterThanUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y1, Y0, Y1
	VPCMPGTB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterThanUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X1, X0, X1
	VPCMPGTB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2LessThanUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y0, Y1, Y1
	VPCMPGTB Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXLessThanUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X0, X1, X1
	VPCMPGTB X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2GreaterEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y0, Y1, Y1
	VPCMPGTB Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTL BX
	NOTL SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X0, X1, X1
	VPCMPGTB X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2LesserEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y1, Y0, Y1
	VPCMPGTB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTL BX
	NOTL SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsUint8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXLesserEqualsUint8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X1, X0, X1
	VPCMPGTB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQW Y1, Y0, Y1
	VPCMPEQW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQW X1, X0, X1
	VPCMPEQW X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y1, Y0, Y1
	VPCMPGTW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterThanUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X1, X0, X1
	VPCMPGTW X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y0, Y1, Y1
	VPCMPGTW Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLessThanUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X0, X1, X1
	VPCMPGTW X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y0, Y1, Y1
	VPCMPGTW Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X0, X1, X1
	VPCMPGTW X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y1, Y0, Y1
	VPCMPGTW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsUint16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLesserEqualsUint16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X1, X0, X1
	VPCMPGTW X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQD Y1, Y0, Y1
	VPCMPEQD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQD X1, X0, X1
	VPCMPEQD X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x00, Y1, Y0, Y1
	VCMPPS $0x00, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x00, X1, X0, X1
	VCMPPS $0x00, X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y1, Y0, Y1
	VPCMPGTD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterThanUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X1, X0, X1
	VPCMPGTD X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x1e, Y1, Y0, Y1
	VCMPPS $0x1e, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterThanFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x1e, X1, X0, X1
	VCMPPS $0x1e, X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y0, Y1, Y1
	VPCMPGTD Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLessThanUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X0, X1, X1
	VPCMPGTD X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x11, Y1, Y0, Y1
	VCMPPS $0x11, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLessThanFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x11, X1, X0, X1
	VCMPPS $0x11, X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y0, Y1, Y1
	VPCMPGTD Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X0, X1, X1
	VPCMPGTD X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x1d, Y1, Y0, Y1
	VCMPPS $0x1d, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x1d, X1, X0, X1
	VCMPPS $0x1d, X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y1, Y0, Y1
	VPCMPGTD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsUint32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLesserEqualsUint32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X1, X0, X1
	VPCMPGTD X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x12, Y1, Y0, Y1
	VCMPPS $0x12, Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsFloat32(dstMask []byte, b float32, rows []float32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLesserEqualsFloat32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x12, X1, X0, X1
	VCMPPS $0x12, X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQQ Y1, Y0, Y1
	VPCMPEQQ Y2, Y0, Y2
	VPCMPEQQ Y3, Y0, Y3
	VPCMPEQQ Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQQ X1, X0, X1
	VPCMPEQQ X2, X0, X2
	VPCMPEQQ X3, X0, X3
	VPCMPEQQ X4, X0, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2EqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2EqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x00, Y1, Y0, Y1
	VCMPPD $0x00, Y2, Y0, Y2
	VCMPPD $0x00, Y3, Y0, Y3
	VCMPPD $0x00, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x00, X1, X0, X1
	VCMPPD $0x00, X2, X0, X2
	VCMPPD $0x00, X3, X0, X3
	VCMPPD $0x00, X4, X0, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y1, Y0, Y1
	VPCMPGTQ Y2, Y0, Y2
	VPCMPGTQ Y3, Y0, Y3
	VPCMPGTQ Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterThanUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ X1, X0, X1
	VPCMPGTQ X2, X0, X2
	VPCMPGTQ X3, X0, X3
	VPCMPGTQ X4, X0, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterThanFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThanFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x1e, Y1, Y0, Y1
	VCMPPD $0x1e, Y2, Y0, Y2
	VCMPPD $0x1e, Y3, Y0, Y3
	VCMPPD $0x1e, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterThanFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterThanFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x1e, X1, X0, X1
	VCMPPD $0x1e, X2, X0, X2
	VCMPPD $0x1e, X3, X0, X3
	VCMPPD $0x1e, X4, X0, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y0, Y1, Y1
	VPCMPGTQ Y0, Y2, Y2
	VPCMPGTQ Y0, Y3, Y3
	VPCMPGTQ Y0, Y4, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLessThanUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ X0, X1, X1
	VPCMPGTQ X0, X2, X2
	VPCMPGTQ X0, X3, X3
	VPCMPGTQ X0, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LessThanFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThanFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x11, Y1, Y0, Y1
	VCMPPD $0x11, Y2, Y0, Y2
	VCMPPD $0x11, Y3, Y0, Y3
	VCMPPD $0x11, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLessThanFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLessThanFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x11, X1, X0, X1
	VCMPPD $0x11, X2, X0, X2
	VCMPPD $0x11, X3, X0, X3
	VCMPPD $0x11, X4, X0, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y0, Y1, Y1
	VPCMPGTQ Y0, Y2, Y2
	VPCMPGTQ Y0, Y3, Y3
	VPCMPGTQ Y0, Y4, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	NOTB DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ X0, X1, X1
	VPCMPGTQ X0, X2, X2
	VPCMPGTQ X0, X3, X3
	VPCMPGTQ X0, X4, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2GreaterEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x1d, Y1, Y0, Y1
	VCMPPD $0x1d, Y2, Y0, Y2
	VCMPPD $0x1d, Y3, Y0, Y3
	VCMPPD $0x1d, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXGreaterEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x1d, X1, X0, X1
	VCMPPD $0x1d, X2, X0, X2
	VCMPPD $0x1d, X3, X0, X3
	VCMPPD $0x1d, X4, X0, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y1, Y0, Y1
	VPCMPGTQ Y2, Y0, Y2
	VPCMPGTQ Y3, Y0, Y3
	VPCMPGTQ Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	NOTB DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsUint64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLesserEqualsUint64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ X1, X0, X1
	VPCMPGTQ X2, X0, X2
	VPCMPGTQ X3, X0, X3
	VPCMPGTQ X4, X0, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2LesserEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2
	VMOVDQU 64(CX), Y3
	VMOVDQU 96(CX), Y4

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x12, Y1, Y0, Y1
	VCMPPD $0x12, Y2, Y0, Y2
	VCMPPD $0x12, Y3, Y0, Y3
	VCMPPD $0x12, Y4, Y0, Y4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI
	VPMOVMSKB Y3, DI
	VPMOVMSKB Y4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXLesserEqualsFloat64(dstMask []byte, b float64, rows []float64)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLesserEqualsFloat64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVQ    b+24(FP), X0
	VPSHUFB const_seven_through_zero<>+0(SB), X0, X0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2
	VMOVDQU 32(CX), X3
	VMOVDQU 48(CX), X4

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x12, X1, X0, X1
	VCMPPD $0x12, X2, X0, X2
	VCMPPD $0x12, X3, X0, X3
	VCMPPD $0x12, X4, X0, X4

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI
	VPMOVMSKB X3, DI
	VPMOVMSKB X4, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2IsNaNFloat32(dstMask []byte, rows []float32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2IsNaNFloat32(SB), NOSPLIT, $0-48
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+24(FP), CX
	MOVQ rows_len+32(FP), DX

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y0
	VMOVDQU 32(CX), Y1

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x04, Y0, Y0, Y0
	VCMPPS $0x04, Y1, Y1, Y1

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y0, BX
	VPMOVMSKB Y1, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXIsNaNFloat32(dstMask []byte, rows []float32)
// Requires: AVX, BMI2
TEXT ·asmAVXIsNaNFloat32(SB), NOSPLIT, $0-48
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+24(FP), CX
	MOVQ rows_len+32(FP), DX

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X0
	VMOVDQU 16(CX), X1

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPS $0x04, X0, X0, X0
	VCMPPS $0x04, X1, X1, X1

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X0, BX
	VPMOVMSKB X1, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVX2IsNaNFloat64(dstMask []byte, rows []float64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2IsNaNFloat64(SB), NOSPLIT, $0-48
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+24(FP), CX
	MOVQ rows_len+32(FP), DX

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 256-bit chunks into YMM registers
	VMOVDQU (CX), Y0
	VMOVDQU 32(CX), Y1
	VMOVDQU 64(CX), Y2
	VMOVDQU 96(CX), Y3

	// Compare all values in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x04, Y0, Y0, Y0
	VCMPPD $0x04, Y1, Y1, Y1
	VCMPPD $0x04, Y2, Y2, Y2
	VCMPPD $0x04, Y3, Y3, Y3

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y0, BX
	VPMOVMSKB Y1, SI
	VPMOVMSKB Y2, DI
	VPMOVMSKB Y3, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)
	SHLB $0x04, R8
	ORB  R8, DI
	MOVB DI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000080, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	VZEROALL
	RET

// func asmAVXIsNaNFloat64(dstMask []byte, rows []float64)
// Requires: AVX, BMI2
TEXT ·asmAVXIsNaNFloat64(SB), NOSPLIT, $0-48
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+24(FP), CX
	MOVQ rows_len+32(FP), DX

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), R9

loop:
	// Load 4 128-bit chunks into XMM registers
	VMOVDQU (CX), X0
	VMOVDQU 16(CX), X1
	VMOVDQU 32(CX), X2
	VMOVDQU 48(CX), X3

	// Compare all values in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VCMPPD $0x04, X0, X0, X0
	VCMPPD $0x04, X1, X1, X1
	VCMPPD $0x04, X2, X2, X2
	VCMPPD $0x04, X3, X3, X3

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X0, BX
	VPMOVMSKB X1, SI
	VPMOVMSKB X2, DI
	VPMOVMSKB X3, R8

	// Drop every second-seventh bit from these registers
	PEXTL R9, BX, BX
	PEXTL R9, SI, SI
	PEXTL R9, DI, DI
	PEXTL R9, R8, R8

	// Each register contains 2 bits, so we first combine them back into bytes before writing them back
	SHLB $0x02, SI
	SHLB $0x04, DI
	SHLB $0x06, R8
	ORB  SI, BL
	ORB  R8, DI
	ORB  DI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	VZEROALL
	RET