Print detailed error message on exception in get_start_timestamp_for_gpu_op

[TaekyungHeo]

Summary

Print detailed error message on exception in get_start_timestamp_for_gpu_op

Test Plan

Previous Version

Traceback (most recent call last):                                                                                                                                                                                                     
  File "/Users/theo/venv/bin/chakra_trace_link", line 8, in <module>                                                                                                                                                                   
    sys.exit(main())                                                                                                                                                                                                                   
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_link.py", line 37, in main       
    linker.link_traces()                                                                                                                                                                                                               
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 473, in link_traces                                                                                                                 
    self.map_pytorch_to_kineto_ops()                                                                                                                                                                                                   
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 542, in map_pytorch_to_kineto_ops                                                                                                   
    cpu_ev_idx_to_gpu_ops_map = self.group_gpu_ops_by_cpu_launchers()                                                                                                                                                                  
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 586, in group_gpu_ops_by_cpu_launchers
    parent_cpu_op = self.find_parent_cpu_op(gpu_op)                                                                                                                                                                                    
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 635, in find_parent_cpu_op                                                                                                          
    kineto_gpu_op.timestamp = self.get_start_timestamp_for_gpu_op(kineto_gpu_op)                                                                                                                                                       
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 668, in get_start_timestamp_for_gpu_op                                                                                              
    raise RuntimeError(f"No valid timestamp found for GPU operator: {kineto_gpu_op.name}")                                                                                                                                             
RuntimeError: No valid timestamp found for GPU operator: void at::native::vectorized_elementwise_kernel<4, at::native::BinaryFunctor<c10::BFloat16, c10::BFloat16, c10::BFloat16, at::native::binary_internal::MulFunctor<float> >, at:
:detail::Array<char*, 3> >(int, at::native::BinaryFunctor<c10::BFloat16, c10::BFloat16, c10::BFloat16, at::native::binary_internal::MulFunctor<float> >, at::detail::Array<char*, 3>)

Current Version

Traceback (most recent call last):
  File "/Users/theo/venv/bin/chakra_trace_link", line 8, in <module>
    sys.exit(main())
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_link.py", line 37, in main
    linker.link_traces()
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 414, in link_traces
    self.map_pytorch_to_kineto_ops()
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 480, in map_pytorch_to_kineto_ops
    cpu_ev_idx_to_gpu_ops_map = self.group_gpu_ops_by_cpu_launchers()
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 520, in group_gpu_ops_by_cpu_launchers
    parent_cpu_op = self.find_parent_cpu_op(gpu_op)
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 566, in find_parent_cpu_op
    kineto_gpu_op.timestamp = self.get_start_timestamp_for_gpu_op(kineto_gpu_op)
  File "/Users/theo/venv/lib/python3.10/site-packages/chakra/src/trace_link/trace_linker.py", line 597, in get_start_timestamp_for_gpu_op
    raise RuntimeError(f"No valid timestamp found for GPU operator: {kineto_gpu_op}")
RuntimeError: No valid timestamp found for GPU operator: KinetoOperator(id=None, category=kernel, name=void at::native::vectorized_elementwise_kernel<4, at::native::BinaryFunctor<c10::BFloat16, c10::BFloat16, c10::BFloat16, at::native::binary_internal::MulFunctor<float> >, at::detail::Array<char*, 3> >(int, at::native::BinaryFunctor<c10::BFloat16, c10::BFloat16, c10::BFloat16, at::native::binary_internal::MulFunctor<float> >, at::detail::Array<char*, 3>), phase=X, inclusive_dur=131, exclusive_dur=131, timestamp=1716386233636001, external_id=148356, ev_idx=-1, tid=2398109, parent_pytorch_op_id=None, inter_thread_dep=None, stream=7, rf_id=None, correlation=1213416)

[github-actions]

MLCommons CLA bot All contributors have signed the MLCommons CLA ✍️ ✅