DXCompiler segfaults when it's impossible to allocate clip/cull registers

[giomasce]

Description
DXCompiler segfaults when trying to compile shaders in which it's impossible to allocate clip/cull registers.

Steps to Reproduce
Compile this shader with DXC (profile vs_6_0):

struct vs_out
{
    float4 position : SV_Position;
    float clip[3] : SV_ClipDistance;
};

void main(uint id : SV_VertexID, out vs_out o)
{
    o.position = float4(0.0f, 0.0f, 0.0f, 1.0f);
    o.clip[0] = 1.0f;
    o.clip[1] = 1.0f;
    o.clip[2] = 1.0f;
}

The compiler will segfault.

Also see on Compiler Explorer.

Other shaders will cause the same behavior:

struct vs_out
{
    float4 position : SV_Position;
    float clip[2] : SV_ClipDistance1;
};

void main(uint id : SV_VertexID, out vs_out o)
{
    o.position = float4(0.0f, 0.0f, 0.0f, 1.0f);
    o.clip[0] = 1.0f;
    o.clip[1] = 1.0f;
}

static const float2 vertices[3] =
{
    {-1.0f,  1.0f},
    { 3.0f,  1.0f},
    {-1.0f, -3.0f},
};

struct vs_out
{
    float4 position : SV_Position;
    float2 clip : SV_ClipDistance;
    float3 clip2 : SV_ClipDistance1;
    float3 cull : SV_CullDistance;
};

void main(uint id : SV_VertexID, out vs_out o)
{
    const float2 pos = vertices[id];
    o.position = float4(pos, 0.0f, 1.0f);
    o.clip = 1.0f;
    o.clip2 = 1.0f;
    o.cull = 1.0f;
}

static const float2 vertices[3] =
{
    {-1.0f,  1.0f},
    { 3.0f,  1.0f},
    {-1.0f, -3.0f},
};

struct vs_out
{
    float4 position : SV_Position;
    float2 clip : SV_ClipDistance;
    float3 clip2 : SV_ClipDistance1;
    float1 cull : SV_CullDistance;
    float2 cull2 : SV_CullDistance1;
};

void main(uint id : SV_VertexID, out vs_out o)
{
    const float2 pos = vertices[id];
    o.position = float4(pos, 0.0f, 1.0f);
    o.clip = 1.0f;
    o.clip2 = 1.0f;
    o.cull = 1.0f;
    o.cull2 = 1.0f;
}

Actual Behavior
The shader is indeed invalid (you cannot allocate three registers for clip/cull distances), so I expect an error message, but not a segfault. The same applies to the other shaders, except that they fail for different reasons (all related to allocating clip/cull registers).

Environment

• DXC version: libdxcompiler.so: 1.8(dev;4662-416fab6b); libdxil.so: 1.8
• Host Operating System: Debian unstable

[giomasce]

This is the backtrace I can observe:

#0  __pthread_kill_implementation (threadid=<optimized out>, signo=signo@entry=0x6, no_tid=no_tid@entry=0x0) at ./nptl/pthread_kill.c:44
#1  0x00007fbc35a9debf in __pthread_kill_internal (threadid=<optimized out>, signo=0x6) at ./nptl/pthread_kill.c:78
#2  0x00007fbc35a49c82 in __GI_raise (sig=sig@entry=0x6) at ../sysdeps/posix/raise.c:26
#3  0x00007fbc35a3257a in __GI_abort () at ./stdlib/abort.c:100
#4  0x00007fbc3661ebf7 in llvm::llvm_unreachable_internal(char const*, char const*, unsigned int) () from /tmp/libdxcompiler.so
#5  0x00007fbc364586ff in hlsl::HLSignatureLower::AllocateDxilInputOutputs() () from /tmp/libdxcompiler.so
#6  0x00007fbc3645ecfb in hlsl::HLSignatureLower::Run() () from /tmp/libdxcompiler.so
#7  0x00007fbc36411af2 in (anonymous namespace)::DxilGenerationPass::runOnModule(llvm::Module&) () from /tmp/libdxcompiler.so
#8  0x00007fbc37196d7d in llvm::legacy::PassManagerImpl::run(llvm::Module&) () from /tmp/libdxcompiler.so
#9  0x00007fbc366bc584 in clang::EmitBackendOutput(clang::DiagnosticsEngine&, clang::CodeGenOptions const&, clang::TargetOptions const&, clang::LangOptions const&, llvm::StringRef, llvm::Module*, clang::BackendAction, llvm::raw_pwrite_stream*) () from /tmp/libdxcompiler.so
#10 0x00007fbc366aac69 in clang::BackendConsumer::HandleTranslationUnit(clang::ASTContext&) () from /tmp/libdxcompiler.so
#11 0x00007fbc36f8c913 in clang::ParseAST(clang::Sema&, bool, bool) () from /tmp/libdxcompiler.so
#12 0x00007fbc3681f968 in clang::FrontendAction::Execute() () from /tmp/libdxcompiler.so
#13 0x00007fbc361e5e96 in DxcCompiler::Compile(DxcBuffer const*, wchar_t const**, unsigned int, IDxcIncludeHandler*, _GUID const&, void**) () from /tmp/libdxcompiler.so
#14 0x00007fbc361e003c in hlsl::DxcCompilerAdapter::WrapCompile(bool, IDxcBlob*, wchar_t const*, wchar_t const*, wchar_t const*, wchar_t const**, unsigned int, DxcDefine const*, unsigned int, IDxcIncludeHandler*, IDxcOperationResult**, wchar_t**, IDxcBlob**) () from /tmp/libdxcompiler.so
#15 0x00007fbc361e112f in hlsl::DxcCompilerAdapter::CompileWithDebug(IDxcBlob*, wchar_t const*, wchar_t const*, wchar_t const*, wchar_t const**, unsigned int, DxcDefine const*, unsigned int, IDxcIncludeHandler*, IDxcOperationResult**, wchar_t**, IDxcBlob**) () from /tmp/libdxcompiler.so
#16 0x00007fbc361e1bb8 in hlsl::DxcCompilerAdapter::Compile(IDxcBlob*, wchar_t const*, wchar_t const*, wchar_t const*, wchar_t const**, unsigned int, DxcDefine const*, unsigned int, IDxcIncludeHandler*, IDxcOperationResult**) () from /tmp/libdxcompiler.so
#17 0x000000000040f036 in DxcContext::Compile() ()
#18 0x00000000004131eb in dxc::main(int, char const**) ()
#19 0x00007fbc35a33d68 in __libc_start_call_main (main=main@entry=0x409a70 <main>, argc=argc@entry=0x3, argv=argv@entry=0x7ffcd6f8e278) at ../sysdeps/nptl/libc_start_call_main.h:58
#20 0x00007fbc35a33e25 in __libc_start_main_impl (main=0x409a70 <main>, argc=0x3, argv=0x7ffcd6f8e278, init=<optimized out>, fini=<optimized out>, rtld_fini=<optimized out>, stack_end=0x7ffcd6f8e268) at ../csu/libc-start.c:360
#21 0x00000000004099a5 in _start ()

On Windows, using the same DXC version, the crash doesn't happen, even if a message is printed that suggests that the same unreachable code is hit. I don't have a Windows machine right now to copy the error message.

[llvm-beanz]

This is a regression introduced in #4599