From 59a81c94c50a63cabdef39a8e03687cac160747e Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Fri, 4 Oct 2019 12:25:17 -0500 Subject: [PATCH] Error-handling as a struct, rather than just a string. (#13) The `Error` struct can carry a lot of information into error messages for C++, but Numba can only raise serialized and globally registered exceptions. Therefore, the error messages from Numba have to be known at compile-time, which severely limits their specificity. Users with unhelpful error messages will have to re-run their code outside of Numba. * Foot-in-the-door to start implementing error-handling as a struct with feedback about the index position of the error (and therefore the Identity, too, if that's available). * Try to initiate a build from the new repository owner (scikit-hep). * Make sure I can still trigger builds. * Add classname() as a virtual method on all Content classes * Changed Error from a 'const char*' to a struct. * The 'failure' function didn't return an error (thanks, Windows compiler). * Start methodically testing error messages. * NumpyArray error messages are correct. * Have to carry knowledge of fake dimensions to give the right error message. * All cpu-kernels should return Error objects, to ensure consistency in the future. * Locations can only be used if there are Identities. * The 'fake' argument isn't necessary because fake dimensions will never have Identities. * These are good error messages. * Update the Numba side in preparation for handling errors. * Numba can raise errors in response to the Error.str value. * Raise errors in Numba, if the function we call returns them (too bad we can't have more specific messages, though). * Put the 'str' member first so that success vs failure can be determined without an offset. * Change order of 'failure' function options to highlight the leading role of the static string. --- VERSION_INFO | 2 +- awkward1/_numba/content.py | 4 +- awkward1/_numba/cpu.py | 34 ++- awkward1/_numba/listarray.py | 3 +- awkward1/_numba/util.py | 25 ++- include/awkward/Content.h | 1 + include/awkward/Identity.h | 4 + include/awkward/ListArray.h | 1 + include/awkward/ListOffsetArray.h | 1 + include/awkward/NumpyArray.h | 1 + include/awkward/RawArray.h | 3 +- include/awkward/cpu-kernels/getitem.h | 43 ++-- include/awkward/cpu-kernels/identity.h | 6 +- include/awkward/cpu-kernels/util.h | 12 +- include/awkward/util.h | 6 +- src/cpu-kernels/getitem.cpp | 172 ++++++++------- src/cpu-kernels/identity.cpp | 18 +- src/cpu-kernels/util.cpp | 23 ++ src/libawkward/Content.cpp | 13 +- src/libawkward/Identity.cpp | 50 ++++- src/libawkward/Index.cpp | 2 +- src/libawkward/ListArray.cpp | 100 +++++---- src/libawkward/ListOffsetArray.cpp | 80 ++++--- src/libawkward/NumpyArray.cpp | 86 +++++--- src/libawkward/util.cpp | 32 +++ tests/test_PR12_listarray_in_numba.py | 4 +- tests/test_PR13_error_handling_struct.py | 269 +++++++++++++++++++++++ 27 files changed, 759 insertions(+), 236 deletions(-) create mode 100644 src/cpu-kernels/util.cpp create mode 100644 src/libawkward/util.cpp create mode 100644 tests/test_PR13_error_handling_struct.py diff --git a/VERSION_INFO b/VERSION_INFO index 1a030947e8..9767cc98e7 100644 --- a/VERSION_INFO +++ b/VERSION_INFO @@ -1 +1 @@ -0.1.9 +0.1.10 diff --git a/awkward1/_numba/content.py b/awkward1/_numba/content.py index f6ea7c4828..47d7b7c2a0 100644 --- a/awkward1/_numba/content.py +++ b/awkward1/_numba/content.py @@ -8,7 +8,9 @@ from .._numba import cpu, util, identity class ContentType(numba.types.Type): - pass + @property + def shortname(self): + return self.name[:self.name.index("Type")] @numba.typing.templates.infer_global(len) class type_len(numba.typing.templates.AbstractTemplate): diff --git a/awkward1/_numba/cpu.py b/awkward1/_numba/cpu.py index db876ff1bb..b56c90e264 100644 --- a/awkward1/_numba/cpu.py +++ b/awkward1/_numba/cpu.py @@ -19,6 +19,26 @@ kernels = ctypes.cdll.LoadLibrary(libpath) +class ErrorType(numba.types.Type): + def __init__(self): + super(ErrorType, self).__init__(name="awkward._numba.cpu.ErrorType") + +class Error(ctypes.Structure): + _fields_ = [("str", ctypes.c_char_p), + ("location", ctypes.c_int64), + ("attempt", ctypes.c_int64), + ("extra", ctypes.c_int64)] + numbatpe = ErrorType() + +@numba.extending.register_model(ErrorType) +class ErrorModel(numba.datamodel.models.StructModel): + def __init__(self, dmm, fe_type): + members = [("str", numba.intp), + ("location", numba.int64), + ("attempt", numba.int64), + ("extra", numba.int64)] + super(ErrorModel, self).__init__(dmm, fe_type, members) + h2ctypes = { "bool": ctypes.c_uint8, "uint8_t *": ctypes.POINTER(ctypes.c_uint8), @@ -29,7 +49,7 @@ "int64_t": ctypes.c_int64, "int64_t *": ctypes.POINTER(ctypes.c_int64), "const int64_t *": ctypes.POINTER(ctypes.c_int64), - "Error": ctypes.c_void_p, + "Error": Error, "void": None, } @@ -41,7 +61,13 @@ if hasattr(kernels, name): rettype = xfcn.find("./type").text params = [(x.find("./declname").text, x.find("./type").text) for x in xfcn.findall("./param")] - getattr(kernels, name).argtypes = [h2ctypes[t] for n, t in params] - getattr(kernels, name).restype = h2ctypes[rettype] getattr(kernels, name).name = name - getattr(kernels, name).numbatpe = numba.typing.ctypes_utils.make_function_type(getattr(kernels, name)) + getattr(kernels, name).argtypes = [h2ctypes[t] for n, t in params] + if rettype == "Error": + getattr(kernels, name).restype = None + tmp = numba.typing.ctypes_utils.make_function_type(getattr(kernels, name)) + getattr(kernels, name).numbatpe = numba.types.functions.ExternalFunctionPointer(Error.numbatpe(*tmp.sig.args), tmp.get_pointer, cconv=tmp.cconv) + getattr(kernels, name).restype = h2ctypes[rettype] + else: + getattr(kernels, name).restype = h2ctypes[rettype] + getattr(kernels, name).numbatpe = numba.typing.ctypes_utils.make_function_type(getattr(kernels, name)) diff --git a/awkward1/_numba/listarray.py b/awkward1/_numba/listarray.py index ade22ce50d..294174d3d6 100644 --- a/awkward1/_numba/listarray.py +++ b/awkward1/_numba/listarray.py @@ -293,7 +293,8 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval, context.get_constant(numba.int64, 0), lenstarts, lenflathead, - lencontent)) + lencontent), + "in {}, indexing error".format(arraytpe.shortname)) nexttpe = arraytpe.contenttpe.carry() nextval = arraytpe.contenttpe.lower_carry(context, builder, arraytpe.contenttpe, util.index64tpe, proxyin.content, nextcarry) diff --git a/awkward1/_numba/util.py b/awkward1/_numba/util.py index dc1bb91bb5..b60052bb4a 100644 --- a/awkward1/_numba/util.py +++ b/awkward1/_numba/util.py @@ -2,6 +2,9 @@ import numpy import numba +import llvmlite.ir.types + +from .._numba import cpu RefType = numba.int64 @@ -28,12 +31,30 @@ def arraylen(context, builder, tpe, val, totpe=None): else: return cast(context, builder, numba.intp, totpe, out) -def call(context, builder, fcn, args): +def call(context, builder, fcn, args, errormessage=None): fcntpe = context.get_function_pointer_type(fcn.numbatpe) fcnval = context.add_dynamic_addr(builder, fcn.numbatpe.get_pointer(fcn), info=fcn.name) fcnptr = builder.bitcast(fcnval, fcntpe) + err = context.call_function_pointer(builder, fcnptr, args) - # FIXME: handle error + + if fcn.restype is cpu.Error: + assert errormessage is not None, "this function can return an error" + proxyerr = numba.cgutils.create_struct_proxy(cpu.Error.numbatpe)(context, builder, value=err) + with builder.if_then(builder.icmp_signed("!=", proxyerr.str, context.get_constant(numba.intp, 0)), likely=False): + context.call_conv.return_user_exc(builder, ValueError, (errormessage,)) + + # pyapi = context.get_python_api(builder) + # exc = pyapi.serialize_object(ValueError(errormessage)) + # excptr = context.call_conv._get_excinfo_argument(builder.function) + # if excptr.name == "excinfo" and excptr.type == llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.struct([llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8)), llvmlite.llvmpy.core.Type.int(32)]))): + # builder.store(exc, excptr) + # builder.ret(numba.targets.callconv.RETCODE_USEREXC) + # elif excptr.name == "py_args" and excptr.type == llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8)): + # pyapi.raise_object(exc) + # builder.ret(llvmlite.llvmpy.core.Constant.null(context.get_value_type(numba.types.pyobject))) + # else: + # raise AssertionError("unrecognized exception calling convention: {}".format(excptr)) def newindex64(context, builder, lentpe, lenval): return numba.targets.arrayobj.numpy_empty_nd(context, builder, index64tpe(lentpe), (lenval,)) diff --git a/include/awkward/Content.h b/include/awkward/Content.h index fd6ae010ef..0ea6c5ce4c 100644 --- a/include/awkward/Content.h +++ b/include/awkward/Content.h @@ -10,6 +10,7 @@ namespace awkward { class Content { public: + virtual const std::string classname() const = 0; virtual const std::shared_ptr id() const = 0; virtual void setid() = 0; virtual void setid(const std::shared_ptr id) = 0; diff --git a/include/awkward/Identity.h b/include/awkward/Identity.h index 482ee2d617..631c051d92 100644 --- a/include/awkward/Identity.h +++ b/include/awkward/Identity.h @@ -32,6 +32,8 @@ namespace awkward { const int64_t width() const { return width_; } const int64_t length() const { return length_; } + virtual const std::string classname() const = 0; + virtual const std::string location(int64_t where) const = 0; virtual const std::shared_ptr to64() const = 0; virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const = 0; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const = 0; @@ -58,6 +60,8 @@ namespace awkward { const std::shared_ptr ptr() const { return ptr_; } + virtual const std::string classname() const; + virtual const std::string location(int64_t where) const; virtual const std::shared_ptr to64() const; virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const; virtual const std::shared_ptr getitem_range(int64_t start, int64_t stop) const; diff --git a/include/awkward/ListArray.h b/include/awkward/ListArray.h index a38f9935b1..9bf296d5b6 100644 --- a/include/awkward/ListArray.h +++ b/include/awkward/ListArray.h @@ -24,6 +24,7 @@ namespace awkward { const IndexOf stops() const { return stops_; } const std::shared_ptr content() const { return content_.get()->shallow_copy(); } + virtual const std::string classname() const; virtual const std::shared_ptr id() const { return id_; } virtual void setid(); virtual void setid(const std::shared_ptr id); diff --git a/include/awkward/ListOffsetArray.h b/include/awkward/ListOffsetArray.h index 8aa7f772d4..a536f853ba 100644 --- a/include/awkward/ListOffsetArray.h +++ b/include/awkward/ListOffsetArray.h @@ -22,6 +22,7 @@ namespace awkward { const IndexOf offsets() const { return offsets_; } const std::shared_ptr content() const { return content_.get()->shallow_copy(); } + virtual const std::string classname() const; virtual const std::shared_ptr id() const { return id_; } virtual void setid(); virtual void setid(const std::shared_ptr id); diff --git a/include/awkward/NumpyArray.h b/include/awkward/NumpyArray.h index e6a5246cec..eb19e82cfa 100644 --- a/include/awkward/NumpyArray.h +++ b/include/awkward/NumpyArray.h @@ -40,6 +40,7 @@ namespace awkward { ssize_t bytelength() const; uint8_t getbyte(ssize_t at) const; + virtual const std::string classname() const; virtual const std::shared_ptr id() const { return id_; } virtual void setid(); virtual void setid(const std::shared_ptr id); diff --git a/include/awkward/RawArray.h b/include/awkward/RawArray.h index 358269a751..34b35152c8 100644 --- a/include/awkward/RawArray.h +++ b/include/awkward/RawArray.h @@ -58,6 +58,7 @@ namespace awkward { T* borrow() const { return borrow(0); } T* borrow(int64_t at) const { return reinterpret_cast(reinterpret_cast(ptr_.get()) + (ssize_t)itemsize_*(ssize_t)(offset_ + at)); } + virtual const std::string classname() const { return std::string("RawArrayOf<") + std::string(typeid(T).name()) + std::string(">"); } virtual const std::shared_ptr id() const { return id_; } virtual void setid() { if (length() <= kMaxInt32) { @@ -138,7 +139,7 @@ namespace awkward { std::shared_ptr nexthead = where.head(); Slice nexttail = where.tail(); Index64 nextadvanced(0); - return getitem_next(nexthead, nexttail, nextadvanced); + return getitem_next(nexthead, nexttail, nextadvanced, false); } const std::shared_ptr getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { if (tail.length() != 0) { diff --git a/include/awkward/cpu-kernels/getitem.h b/include/awkward/cpu-kernels/getitem.h index 892f865e63..9e7dccac10 100644 --- a/include/awkward/cpu-kernels/getitem.h +++ b/include/awkward/cpu-kernels/getitem.h @@ -9,37 +9,38 @@ extern "C" { void awkward_regularize_rangeslice(int64_t* start, int64_t* stop, bool posstep, bool hasstart, bool hasstop, int64_t length); Error awkward_regularize_arrayslice_64(int64_t* flatheadptr, int64_t lenflathead, int64_t length); - void awkward_slicearray_ravel_64(int64_t* toptr, const int64_t* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides); + Error awkward_slicearray_ravel_64(int64_t* toptr, const int64_t* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides); - void awkward_carry_arange_64(int64_t* toptr, int64_t length); + Error awkward_carry_arange_64(int64_t* toptr, int64_t length); Error awkward_identity32_getitem_carry_64(int32_t* newidentityptr, const int32_t* identityptr, const int64_t* carryptr, int64_t lencarry, int64_t offset, int64_t width, int64_t length); Error awkward_identity64_getitem_carry_64(int64_t* newidentityptr, const int64_t* identityptr, const int64_t* carryptr, int64_t lencarry, int64_t offset, int64_t width, int64_t length); - void awkward_numpyarray_contiguous_init_64(int64_t* toptr, int64_t skip, int64_t stride); - void awkward_numpyarray_contiguous_copy_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos); - void awkward_numpyarray_contiguous_next_64(int64_t* topos, const int64_t* frompos, int64_t len, int64_t skip, int64_t stride); - void awkward_numpyarray_getitem_next_null_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos); - void awkward_numpyarray_getitem_next_at_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t skip, int64_t at); - void awkward_numpyarray_getitem_next_range_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step); - void awkward_numpyarray_getitem_next_range_advanced_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step); - void awkward_numpyarray_getitem_next_array_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip); - void awkward_numpyarray_getitem_next_array_advanced_64(int64_t* nextcarryptr, const int64_t* carryptr, const int64_t* advancedptr, const int64_t* flatheadptr, int64_t lencarry, int64_t skip); + Error awkward_numpyarray_contiguous_init_64(int64_t* toptr, int64_t skip, int64_t stride); + Error awkward_numpyarray_contiguous_copy_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos); + Error awkward_numpyarray_contiguous_next_64(int64_t* topos, const int64_t* frompos, int64_t len, int64_t skip, int64_t stride); + + Error awkward_numpyarray_getitem_next_null_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos); + Error awkward_numpyarray_getitem_next_at_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t skip, int64_t at); + Error awkward_numpyarray_getitem_next_range_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step); + Error awkward_numpyarray_getitem_next_range_advanced_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step); + Error awkward_numpyarray_getitem_next_array_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip); + Error awkward_numpyarray_getitem_next_array_advanced_64(int64_t* nextcarryptr, const int64_t* carryptr, const int64_t* advancedptr, const int64_t* flatheadptr, int64_t lencarry, int64_t skip); Error awkward_listarray32_getitem_next_at_64(int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t at); Error awkward_listarray64_getitem_next_at_64(int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t at); - void awkward_listarray32_getitem_next_range_carrylength(int64_t* carrylength, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step); - void awkward_listarray64_getitem_next_range_carrylength(int64_t* carrylength, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step); + Error awkward_listarray32_getitem_next_range_carrylength(int64_t* carrylength, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step); + Error awkward_listarray64_getitem_next_range_carrylength(int64_t* carrylength, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step); - void awkward_listarray32_getitem_next_range_64(int32_t* tooffsets, int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step); - void awkward_listarray64_getitem_next_range_64(int64_t* tooffsets, int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step); + Error awkward_listarray32_getitem_next_range_64(int32_t* tooffsets, int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step); + Error awkward_listarray64_getitem_next_range_64(int64_t* tooffsets, int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step); - void awkward_listarray32_getitem_next_range_counts_64(int64_t* total, const int32_t* fromoffsets, int64_t lenstarts); - void awkward_listarray64_getitem_next_range_counts_64(int64_t* total, const int64_t* fromoffsets, int64_t lenstarts); + Error awkward_listarray32_getitem_next_range_counts_64(int64_t* total, const int32_t* fromoffsets, int64_t lenstarts); + Error awkward_listarray64_getitem_next_range_counts_64(int64_t* total, const int64_t* fromoffsets, int64_t lenstarts); - void awkward_listarray32_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int32_t* fromoffsets, int64_t lenstarts); - void awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromoffsets, int64_t lenstarts); + Error awkward_listarray32_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int32_t* fromoffsets, int64_t lenstarts); + Error awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromoffsets, int64_t lenstarts); Error awkward_listarray32_getitem_next_array_64(int32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); Error awkward_listarray64_getitem_next_array_64(int64_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); @@ -47,8 +48,8 @@ extern "C" { Error awkward_listarray32_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); Error awkward_listarray64_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent); - void awkward_listarray32_getitem_carry_64(int32_t* tostarts, int32_t* tostops, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lencarry); - void awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lencarry); + Error awkward_listarray32_getitem_carry_64(int32_t* tostarts, int32_t* tostops, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry); + Error awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry); } #endif // AWKWARDCPU_GETITEM_H_ diff --git a/include/awkward/cpu-kernels/identity.h b/include/awkward/cpu-kernels/identity.h index f7f37d1d84..3dbe706dac 100644 --- a/include/awkward/cpu-kernels/identity.h +++ b/include/awkward/cpu-kernels/identity.h @@ -6,10 +6,10 @@ #include "awkward/cpu-kernels/util.h" extern "C" { - void awkward_new_identity32(int32_t* toptr, int64_t length); - void awkward_new_identity64(int64_t* toptr, int64_t length); + Error awkward_new_identity32(int32_t* toptr, int64_t length); + Error awkward_new_identity64(int64_t* toptr, int64_t length); - void awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length); + Error awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length); Error awkward_identity32_from_listarray32(int32_t* toptr, const int32_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); Error awkward_identity64_from_listarray32(int64_t* toptr, const int64_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth); diff --git a/include/awkward/cpu-kernels/util.h b/include/awkward/cpu-kernels/util.h index 36f560dc12..cc3a5af1d5 100644 --- a/include/awkward/cpu-kernels/util.h +++ b/include/awkward/cpu-kernels/util.h @@ -20,9 +20,17 @@ #include #endif +#include + extern "C" { - typedef const char* Error; - const Error kNoError = nullptr; + struct Error { + const char* str; + int64_t location; + int64_t attempt; + int64_t extra; + }; + Error success(); + Error failure(const char* str, int64_t location, int64_t attempt); const int8_t kMaxInt8 = 127; // 2**7 - 1 const uint8_t kMaxUInt8 = 255; // 2**8 - 1 diff --git a/include/awkward/util.h b/include/awkward/util.h index 962812935e..c8c5d1cf22 100644 --- a/include/awkward/util.h +++ b/include/awkward/util.h @@ -3,14 +3,16 @@ #ifndef AWKWARD_UTIL_H_ #define AWKWARD_UTIL_H_ -#include +#include #include "awkward/cpu-kernels/util.h" namespace awkward { - #define HANDLE_ERROR(err) { if (err != kNoError) { throw std::invalid_argument(err); } } + class Identity; namespace util { + void handle_error(const Error& err, const std::string classname, const Identity* id); + template class array_deleter { public: diff --git a/src/cpu-kernels/getitem.cpp b/src/cpu-kernels/getitem.cpp index ed36351c6d..b9ec3e8e2a 100644 --- a/src/cpu-kernels/getitem.cpp +++ b/src/cpu-kernels/getitem.cpp @@ -35,21 +35,22 @@ void awkward_regularize_rangeslice(int64_t* start, int64_t* stop, bool posstep, template Error awkward_regularize_arrayslice(T* flatheadptr, int64_t lenflathead, int64_t length) { for (int64_t i = 0; i < lenflathead; i++) { + T original = flatheadptr[i]; if (flatheadptr[i] < 0) { flatheadptr[i] += length; } if (flatheadptr[i] < 0 || flatheadptr[i] >= length) { - return "index out of range"; + return failure("index out of range", kSliceNone, original); } } - return kNoError; + return success(); } Error awkward_regularize_arrayslice_64(int64_t* flatheadptr, int64_t lenflathead, int64_t length) { return awkward_regularize_arrayslice(flatheadptr, lenflathead, length); } template -void awkward_slicearray_ravel(T* toptr, const T* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides) { +Error awkward_slicearray_ravel(T* toptr, const T* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides) { if (ndim == 1) { for (T i = 0; i < shape[0]; i++) { toptr[i] = fromptr[i*strides[0]]; @@ -57,35 +58,40 @@ void awkward_slicearray_ravel(T* toptr, const T* fromptr, int64_t ndim, const in } else { for (T i = 0; i < shape[0]; i++) { - awkward_slicearray_ravel(&toptr[i*shape[1]], &fromptr[i*strides[0]], ndim - 1, &shape[1], &strides[1]); + Error err = awkward_slicearray_ravel(&toptr[i*shape[1]], &fromptr[i*strides[0]], ndim - 1, &shape[1], &strides[1]); + if (err.str != nullptr) { + return err; + } } } + return success(); } -void awkward_slicearray_ravel_64(int64_t* toptr, const int64_t* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides) { - awkward_slicearray_ravel(toptr, fromptr, ndim, shape, strides); +Error awkward_slicearray_ravel_64(int64_t* toptr, const int64_t* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides) { + return awkward_slicearray_ravel(toptr, fromptr, ndim, shape, strides); } template -void awkward_carry_arange(T* toptr, int64_t length) { +Error awkward_carry_arange(T* toptr, int64_t length) { for (int64_t i = 0; i < length; i++) { toptr[i] = i; } + return success(); } -void awkward_carry_arange_64(int64_t* toptr, int64_t length) { - awkward_carry_arange(toptr, length); +Error awkward_carry_arange_64(int64_t* toptr, int64_t length) { + return awkward_carry_arange(toptr, length); } template Error awkward_identity_getitem_carry(ID* newidentityptr, const ID* identityptr, const T* carryptr, int64_t lencarry, int64_t offset, int64_t width, int64_t length) { for (int64_t i = 0; i < lencarry; i++) { if (carryptr[i] >= length) { - return "index out of range for identity"; + return failure("index out of range", kSliceNone, carryptr[i]); } for (int64_t j = 0; j < width; j++) { newidentityptr[width*i + j] = identityptr[offset + width*carryptr[i] + j]; } } - return kNoError; + return success(); } Error awkward_identity32_getitem_carry_64(int32_t* newidentityptr, const int32_t* identityptr, const int64_t* carryptr, int64_t lencarry, int64_t offset, int64_t width, int64_t length) { return awkward_identity_getitem_carry(newidentityptr, identityptr, carryptr, lencarry, offset, width, length); @@ -95,103 +101,112 @@ Error awkward_identity64_getitem_carry_64(int64_t* newidentityptr, const int64_t } template -void awkward_numpyarray_contiguous_init(T* toptr, int64_t skip, int64_t stride) { +Error awkward_numpyarray_contiguous_init(T* toptr, int64_t skip, int64_t stride) { for (int64_t i = 0; i < skip; i++) { toptr[i] = i*stride; } + return success(); } -void awkward_numpyarray_contiguous_init_64(int64_t* toptr, int64_t skip, int64_t stride) { - awkward_numpyarray_contiguous_init(toptr, skip, stride); +Error awkward_numpyarray_contiguous_init_64(int64_t* toptr, int64_t skip, int64_t stride) { + return awkward_numpyarray_contiguous_init(toptr, skip, stride); } template -void awkward_numpyarray_contiguous_copy(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const T* pos) { +Error awkward_numpyarray_contiguous_copy(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const T* pos) { for (int64_t i = 0; i < len; i++) { memcpy(&toptr[i*stride], &fromptr[offset + (int64_t)pos[i]], (size_t)stride); } + return success(); } -void awkward_numpyarray_contiguous_copy_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos) { - awkward_numpyarray_contiguous_copy(toptr, fromptr, len, stride, offset, pos); +Error awkward_numpyarray_contiguous_copy_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos) { + return awkward_numpyarray_contiguous_copy(toptr, fromptr, len, stride, offset, pos); } template -void awkward_numpyarray_contiguous_next(T* topos, const T* frompos, int64_t len, int64_t skip, int64_t stride) { +Error awkward_numpyarray_contiguous_next(T* topos, const T* frompos, int64_t len, int64_t skip, int64_t stride) { for (int64_t i = 0; i < len; i++) { for (int64_t j = 0; j < skip; j++) { topos[i*skip + j] = frompos[i] + j*stride; } } + return success(); } -void awkward_numpyarray_contiguous_next_64(int64_t* topos, const int64_t* frompos, int64_t len, int64_t skip, int64_t stride) { - awkward_numpyarray_contiguous_next(topos, frompos, len, skip, stride); +Error awkward_numpyarray_contiguous_next_64(int64_t* topos, const int64_t* frompos, int64_t len, int64_t skip, int64_t stride) { + return awkward_numpyarray_contiguous_next(topos, frompos, len, skip, stride); } template -void awkward_numpyarray_getitem_next_null(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const T* pos) { +Error awkward_numpyarray_getitem_next_null(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const T* pos) { for (int64_t i = 0; i < len; i++) { std::memcpy(&toptr[i*stride], &fromptr[offset + pos[i]*stride], (size_t)stride); } + return success(); } -void awkward_numpyarray_getitem_next_null_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos) { - awkward_numpyarray_getitem_next_null(toptr, fromptr, len, stride, offset, pos); +Error awkward_numpyarray_getitem_next_null_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos) { + return awkward_numpyarray_getitem_next_null(toptr, fromptr, len, stride, offset, pos); } template -void awkward_numpyarray_getitem_next_at(T* nextcarryptr, const T* carryptr, int64_t lencarry, int64_t skip, int64_t at) { +Error awkward_numpyarray_getitem_next_at(T* nextcarryptr, const T* carryptr, int64_t lencarry, int64_t skip, int64_t at) { for (int64_t i = 0; i < lencarry; i++) { nextcarryptr[i] = skip*carryptr[i] + at; } + return success(); } -void awkward_numpyarray_getitem_next_at_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t skip, int64_t at) { - awkward_numpyarray_getitem_next_at(nextcarryptr, carryptr, lencarry, skip, at); +Error awkward_numpyarray_getitem_next_at_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t skip, int64_t at) { + return awkward_numpyarray_getitem_next_at(nextcarryptr, carryptr, lencarry, skip, at); } template -void awkward_numpyarray_getitem_next_range(T* nextcarryptr, const T* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step) { +Error awkward_numpyarray_getitem_next_range(T* nextcarryptr, const T* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step) { for (int64_t i = 0; i < lencarry; i++) { for (int64_t j = 0; j < lenhead; j++) { nextcarryptr[i*lenhead + j] = skip*carryptr[i] + start + j*step; } } + return success(); } -void awkward_numpyarray_getitem_next_range_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step) { - awkward_numpyarray_getitem_next_range(nextcarryptr, carryptr, lencarry, lenhead, skip, start, step); +Error awkward_numpyarray_getitem_next_range_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step) { + return awkward_numpyarray_getitem_next_range(nextcarryptr, carryptr, lencarry, lenhead, skip, start, step); } template -void awkward_numpyarray_getitem_next_range_advanced(T* nextcarryptr, T* nextadvancedptr, const T* carryptr, const T* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step) { +Error awkward_numpyarray_getitem_next_range_advanced(T* nextcarryptr, T* nextadvancedptr, const T* carryptr, const T* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step) { for (int64_t i = 0; i < lencarry; i++) { for (int64_t j = 0; j < lenhead; j++) { nextcarryptr[i*lenhead + j] = skip*carryptr[i] + start + j*step; nextadvancedptr[i*lenhead + j] = advancedptr[i]; } } + return success(); } -void awkward_numpyarray_getitem_next_range_advanced_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step) { - awkward_numpyarray_getitem_next_range_advanced(nextcarryptr, nextadvancedptr, carryptr, advancedptr, lencarry, lenhead, skip, start, step); +Error awkward_numpyarray_getitem_next_range_advanced_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step) { + return awkward_numpyarray_getitem_next_range_advanced(nextcarryptr, nextadvancedptr, carryptr, advancedptr, lencarry, lenhead, skip, start, step); } template -void awkward_numpyarray_getitem_next_array(T* nextcarryptr, T* nextadvancedptr, const T* carryptr, const T* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip) { +Error awkward_numpyarray_getitem_next_array(T* nextcarryptr, T* nextadvancedptr, const T* carryptr, const T* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip) { for (int64_t i = 0; i < lencarry; i++) { for (int64_t j = 0; j < lenflathead; j++) { nextcarryptr[i*lenflathead + j] = skip*carryptr[i] + flatheadptr[j]; nextadvancedptr[i*lenflathead + j] = j; } } + return success(); } -void awkward_numpyarray_getitem_next_array_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip) { - awkward_numpyarray_getitem_next_array(nextcarryptr, nextadvancedptr, carryptr, flatheadptr, lencarry, lenflathead, skip); +Error awkward_numpyarray_getitem_next_array_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip) { + return awkward_numpyarray_getitem_next_array(nextcarryptr, nextadvancedptr, carryptr, flatheadptr, lencarry, lenflathead, skip); } template -void awkward_numpyarray_getitem_next_array_advanced(T* nextcarryptr, const T* carryptr, const T* advancedptr, const T* flatheadptr, int64_t lencarry, int64_t skip) { +Error awkward_numpyarray_getitem_next_array_advanced(T* nextcarryptr, const T* carryptr, const T* advancedptr, const T* flatheadptr, int64_t lencarry, int64_t skip) { for (int64_t i = 0; i < lencarry; i++) { nextcarryptr[i] = skip*carryptr[i] + flatheadptr[advancedptr[i]]; } + return success(); } -void awkward_numpyarray_getitem_next_array_advanced_64(int64_t* nextcarryptr, const int64_t* carryptr, const int64_t* advancedptr, const int64_t* flatheadptr, int64_t lencarry, int64_t skip) { - awkward_numpyarray_getitem_next_array_advanced(nextcarryptr, carryptr, advancedptr, flatheadptr, lencarry, skip); +Error awkward_numpyarray_getitem_next_array_advanced_64(int64_t* nextcarryptr, const int64_t* carryptr, const int64_t* advancedptr, const int64_t* flatheadptr, int64_t lencarry, int64_t skip) { + return awkward_numpyarray_getitem_next_array_advanced(nextcarryptr, carryptr, advancedptr, flatheadptr, lencarry, skip); } template @@ -203,11 +218,11 @@ Error awkward_listarray_getitem_next_at(T* tocarry, const C* fromstarts, const C regular_at += length; } if (!(0 <= regular_at && regular_at < length)) { - return "index out of range"; + return failure("index out of range", i, at); } tocarry[i] = fromstarts[startsoffset + i] + regular_at; } - return kNoError; + return success(); } Error awkward_listarray32_getitem_next_at_64(int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t at) { return awkward_listarray_getitem_next_at(tocarry, fromstarts, fromstops, lenstarts, startsoffset, stopsoffset, at); @@ -217,7 +232,7 @@ Error awkward_listarray64_getitem_next_at_64(int64_t* tocarry, const int64_t* fr } template -void awkward_listarray_getitem_next_range_carrylength(int64_t* carrylength, const C* fromstarts, const C* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { +Error awkward_listarray_getitem_next_range_carrylength(int64_t* carrylength, const C* fromstarts, const C* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { *carrylength = 0; for (int64_t i = 0; i < lenstarts; i++) { int64_t length = fromstops[stopsoffset + i] - fromstarts[startsoffset + i]; @@ -235,17 +250,17 @@ void awkward_listarray_getitem_next_range_carrylength(int64_t* carrylength, cons } } } + return success(); } -void awkward_listarray32_getitem_next_range_carrylength(int64_t* carrylength, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { - awkward_listarray_getitem_next_range_carrylength(carrylength, fromstarts, fromstops, lenstarts, startsoffset, stopsoffset, start, stop, step); +Error awkward_listarray32_getitem_next_range_carrylength(int64_t* carrylength, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { + return awkward_listarray_getitem_next_range_carrylength(carrylength, fromstarts, fromstops, lenstarts, startsoffset, stopsoffset, start, stop, step); } -void awkward_listarray64_getitem_next_range_carrylength(int64_t* carrylength, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { - awkward_listarray_getitem_next_range_carrylength(carrylength, fromstarts, fromstops, lenstarts, startsoffset, stopsoffset, start, stop, step); +Error awkward_listarray64_getitem_next_range_carrylength(int64_t* carrylength, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { + return awkward_listarray_getitem_next_range_carrylength(carrylength, fromstarts, fromstops, lenstarts, startsoffset, stopsoffset, start, stop, step); } - template -void awkward_listarray_getitem_next_range(C* tooffsets, T* tocarry, const C* fromstarts, const C* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { +Error awkward_listarray_getitem_next_range(C* tooffsets, T* tocarry, const C* fromstarts, const C* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { int64_t k = 0; tooffsets[0] = 0; for (int64_t i = 0; i < lenstarts; i++) { @@ -267,42 +282,45 @@ void awkward_listarray_getitem_next_range(C* tooffsets, T* tocarry, const C* fro } tooffsets[i + 1] = (C)k; } + return success(); } -void awkward_listarray32_getitem_next_range_64(int32_t* tooffsets, int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { - awkward_listarray_getitem_next_range(tooffsets, tocarry, fromstarts, fromstops, lenstarts, startsoffset, stopsoffset, start, stop, step); +Error awkward_listarray32_getitem_next_range_64(int32_t* tooffsets, int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { + return awkward_listarray_getitem_next_range(tooffsets, tocarry, fromstarts, fromstops, lenstarts, startsoffset, stopsoffset, start, stop, step); } -void awkward_listarray64_getitem_next_range_64(int64_t* tooffsets, int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { - awkward_listarray_getitem_next_range(tooffsets, tocarry, fromstarts, fromstops, lenstarts, startsoffset, stopsoffset, start, stop, step); +Error awkward_listarray64_getitem_next_range_64(int64_t* tooffsets, int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step) { + return awkward_listarray_getitem_next_range(tooffsets, tocarry, fromstarts, fromstops, lenstarts, startsoffset, stopsoffset, start, stop, step); } template -void awkward_listarray_getitem_next_range_counts(int64_t* total, const C* fromoffsets, int64_t lenstarts) { +Error awkward_listarray_getitem_next_range_counts(int64_t* total, const C* fromoffsets, int64_t lenstarts) { *total = 0; for (int64_t i = 0; i < lenstarts; i++) { *total = *total + fromoffsets[i + 1] - fromoffsets[i]; } + return success(); } -void awkward_listarray32_getitem_next_range_counts_64(int64_t* total, const int32_t* fromoffsets, int64_t lenstarts) { - awkward_listarray_getitem_next_range_counts(total, fromoffsets, lenstarts); +Error awkward_listarray32_getitem_next_range_counts_64(int64_t* total, const int32_t* fromoffsets, int64_t lenstarts) { + return awkward_listarray_getitem_next_range_counts(total, fromoffsets, lenstarts); } -void awkward_listarray64_getitem_next_range_counts_64(int64_t* total, const int64_t* fromoffsets, int64_t lenstarts) { - awkward_listarray_getitem_next_range_counts(total, fromoffsets, lenstarts); +Error awkward_listarray64_getitem_next_range_counts_64(int64_t* total, const int64_t* fromoffsets, int64_t lenstarts) { + return awkward_listarray_getitem_next_range_counts(total, fromoffsets, lenstarts); } template -void awkward_listarray_getitem_next_range_spreadadvanced(T* toadvanced, const T* fromadvanced, const C* fromoffsets, int64_t lenstarts) { +Error awkward_listarray_getitem_next_range_spreadadvanced(T* toadvanced, const T* fromadvanced, const C* fromoffsets, int64_t lenstarts) { for (int64_t i = 0; i < lenstarts; i++) { C count = fromoffsets[i + 1] - fromoffsets[i]; for (int64_t j = 0; j < count; j++) { toadvanced[fromoffsets[i] + j] = fromadvanced[i]; } } + return success(); } -void awkward_listarray32_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int32_t* fromoffsets, int64_t lenstarts) { - awkward_listarray_getitem_next_range_spreadadvanced(toadvanced, fromadvanced, fromoffsets, lenstarts); +Error awkward_listarray32_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int32_t* fromoffsets, int64_t lenstarts) { + return awkward_listarray_getitem_next_range_spreadadvanced(toadvanced, fromadvanced, fromoffsets, lenstarts); } -void awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromoffsets, int64_t lenstarts) { - awkward_listarray_getitem_next_range_spreadadvanced(toadvanced, fromadvanced, fromoffsets, lenstarts); +Error awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromoffsets, int64_t lenstarts) { + return awkward_listarray_getitem_next_range_spreadadvanced(toadvanced, fromadvanced, fromoffsets, lenstarts); } template @@ -310,10 +328,10 @@ Error awkward_listarray_getitem_next_array(C* tooffsets, T* tocarry, T* toadvanc tooffsets[0] = 0; for (int64_t i = 0; i < lenstarts; i++) { if (fromstops[stopsoffset + i] < fromstarts[startsoffset + i]) { - return "stops[i] < starts[i]"; + return failure("stops[i] < starts[i]", i, kSliceNone); } if (fromstarts[startsoffset + i] != fromstops[stopsoffset + i] && fromstops[stopsoffset + i] > lencontent) { - return "stops[i] > len(content)"; + return failure("stops[i] > len(content)", i, kSliceNone); } int64_t length = fromstops[stopsoffset + i] - fromstarts[startsoffset + i]; for (int64_t j = 0; j < lenarray; j++) { @@ -322,14 +340,14 @@ Error awkward_listarray_getitem_next_array(C* tooffsets, T* tocarry, T* toadvanc regular_at += length; } if (!(0 <= regular_at && regular_at < length)) { - return "array[i] is out of range for at least one sublist"; + return failure("index out of range", i, fromarray[j]); } tocarry[i*lenarray + j] = fromstarts[startsoffset + i] + regular_at; toadvanced[i*lenarray + j] = j; } tooffsets[i + 1] = (C)((i + 1)*lenarray); } - return kNoError; + return success(); } Error awkward_listarray32_getitem_next_array_64(int32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { return awkward_listarray_getitem_next_array(tooffsets, tocarry, toadvanced, fromstarts, fromstops, fromarray, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); @@ -342,26 +360,26 @@ template Error awkward_listarray_getitem_next_array_advanced(T* tocarry, T* toadvanced, const C* fromstarts, const C* fromstops, const T* fromarray, const T* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { for (int64_t i = 0; i < lenstarts; i++) { if (fromstops[stopsoffset + i] < fromstarts[startsoffset + i]) { - return "stops[i] < starts[i]"; + return failure("stops[i] < starts[i]", i, kSliceNone); } if (fromstarts[startsoffset + i] != fromstops[stopsoffset + i] && fromstops[stopsoffset + i] > lencontent) { - return "stops[i] > len(content)"; + return failure("stops[i] > len(content)", i, kSliceNone); } int64_t length = fromstops[stopsoffset + i] - fromstarts[startsoffset + i]; if (fromadvanced[i] >= lenarray) { - return "lengths of advanced indexes must match"; + return failure("lengths of advanced indexes must match", i, kSliceNone); } int64_t regular_at = fromarray[fromadvanced[i]]; if (regular_at < 0) { regular_at += length; } if (!(0 <= regular_at && regular_at < length)) { - return "array[i] is out of range for at least one sublist"; + return failure("index out of range", i, fromarray[fromadvanced[i]]); } tocarry[i] = fromstarts[startsoffset + i] + regular_at; toadvanced[i] = i; } - return kNoError; + return success(); } Error awkward_listarray32_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent) { return awkward_listarray_getitem_next_array_advanced(tocarry, toadvanced, fromstarts, fromstops, fromarray, fromadvanced, startsoffset, stopsoffset, lenstarts, lenarray, lencontent); @@ -371,15 +389,19 @@ Error awkward_listarray64_getitem_next_array_advanced_64(int64_t* tocarry, int64 } template -void awkward_listarray_getitem_carry(C* tostarts, C* tostops, const C* fromstarts, const C* fromstops, const T* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lencarry) { +Error awkward_listarray_getitem_carry(C* tostarts, C* tostops, const C* fromstarts, const C* fromstops, const T* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry) { for (int64_t i = 0; i < lencarry; i++) { + if (fromcarry[i] >= lenstarts) { + return failure("index out of range", i, fromcarry[i]); + } tostarts[i] = (C)(fromstarts[startsoffset + fromcarry[i]]); tostops[i] = (C)(fromstops[stopsoffset + fromcarry[i]]); } + return success(); } -void awkward_listarray32_getitem_carry_64(int32_t* tostarts, int32_t* tostops, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lencarry) { - awkward_listarray_getitem_carry(tostarts, tostops, fromstarts, fromstops, fromcarry, startsoffset, stopsoffset, lencarry); +Error awkward_listarray32_getitem_carry_64(int32_t* tostarts, int32_t* tostops, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry) { + return awkward_listarray_getitem_carry(tostarts, tostops, fromstarts, fromstops, fromcarry, startsoffset, stopsoffset, lenstarts, lencarry); } -void awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lencarry) { - awkward_listarray_getitem_carry(tostarts, tostops, fromstarts, fromstops, fromcarry, startsoffset, stopsoffset, lencarry); +Error awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry) { + return awkward_listarray_getitem_carry(tostarts, tostops, fromstarts, fromstops, fromcarry, startsoffset, stopsoffset, lenstarts, lencarry); } diff --git a/src/cpu-kernels/identity.cpp b/src/cpu-kernels/identity.cpp index d33468eea4..753667efd3 100644 --- a/src/cpu-kernels/identity.cpp +++ b/src/cpu-kernels/identity.cpp @@ -3,22 +3,24 @@ #include "awkward/cpu-kernels/identity.h" template -void awkward_new_identity(T* toptr, int64_t length) { +Error awkward_new_identity(T* toptr, int64_t length) { for (T i = 0; i < length; i++) { toptr[i] = i; } + return success(); } -void awkward_new_identity32(int32_t* toptr, int64_t length) { - awkward_new_identity(toptr, length); +Error awkward_new_identity32(int32_t* toptr, int64_t length) { + return awkward_new_identity(toptr, length); } -void awkward_new_identity64(int64_t* toptr, int64_t length) { - awkward_new_identity(toptr, length); +Error awkward_new_identity64(int64_t* toptr, int64_t length) { + return awkward_new_identity(toptr, length); } -void awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length) { +Error awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length) { for (int64_t i = 0; i < length; i++) { toptr[i]= (int64_t)fromptr[i]; } + return success(); } template @@ -30,7 +32,7 @@ Error awkward_identity_from_listarray(ID* toptr, const ID* fromptr, const T* fro int64_t start = fromstarts[startsoffset + i]; int64_t stop = fromstops[stopsoffset + i]; if (start != stop && stop > tolength) { - return "max(stop) > len(content)"; + return failure("max(stop) > len(content)", i, kSliceNone); } for (int64_t j = start; j < stop; j++) { for (int64_t k = 0; k < fromwidth; k++) { @@ -39,7 +41,7 @@ Error awkward_identity_from_listarray(ID* toptr, const ID* fromptr, const T* fro toptr[j*(fromwidth + 1) + fromwidth] = ID(j - start); } } - return kNoError; + return success(); } Error awkward_identity32_from_listarray32(int32_t* toptr, const int32_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth) { return awkward_identity_from_listarray(toptr, fromptr, fromstarts, fromstops, fromptroffset, startsoffset, stopsoffset, tolength, fromlength, fromwidth); diff --git a/src/cpu-kernels/util.cpp b/src/cpu-kernels/util.cpp new file mode 100644 index 0000000000..4bf2b265cd --- /dev/null +++ b/src/cpu-kernels/util.cpp @@ -0,0 +1,23 @@ +// BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +#include + +#include "awkward/cpu-kernels/util.h" + +Error success() { + Error out; + out.str = nullptr; + out.location = kSliceNone; + out.attempt = kSliceNone; + out.extra = 0; + return out; +} + +Error failure(const char* str, int64_t location, int64_t attempt) { + Error out; + out.str = str; + out.location = location; + out.attempt = attempt; + out.extra = 0; + return out; +} diff --git a/src/libawkward/Content.cpp b/src/libawkward/Content.cpp index aca705dfca..aa92c48a12 100644 --- a/src/libawkward/Content.cpp +++ b/src/libawkward/Content.cpp @@ -1,6 +1,6 @@ // BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE -#include "awkward/ListOffsetArray.h" +#include "awkward/ListArray.h" #include "awkward/Content.h" @@ -10,10 +10,11 @@ namespace awkward { } const std::shared_ptr Content::getitem(const Slice& where) const { - Index64 nextoffsets(2); - nextoffsets.ptr().get()[0] = 0; - nextoffsets.ptr().get()[1] = length(); - ListOffsetArrayOf next(std::shared_ptr(nullptr), nextoffsets, shallow_copy()); + Index64 nextstarts(1); + Index64 nextstops(1); + *nextstarts.ptr().get() = 0; + *nextstops.ptr().get() = length(); + ListArrayOf next(std::shared_ptr(nullptr), nextstarts, nextstops, shallow_copy()); std::shared_ptr nexthead = where.head(); Slice nexttail = where.tail(); @@ -40,7 +41,7 @@ namespace awkward { std::vector> items = { std::shared_ptr(new SliceEllipsis()) }; items.insert(items.end(), tailitems.begin(), tailitems.end()); std::shared_ptr nexthead(new SliceRange(Slice::none(), Slice::none(), 1)); - Slice nexttail(items, true); + Slice nexttail(items); return getitem_next(nexthead, nexttail, advanced); } } diff --git a/src/libawkward/Identity.cpp b/src/libawkward/Identity.cpp index 25f46e4d4a..55472123e7 100644 --- a/src/libawkward/Identity.cpp +++ b/src/libawkward/Identity.cpp @@ -18,6 +18,40 @@ namespace awkward { return numrefs++; } + template + const std::string IdentityOf::classname() const { + if (std::is_same::value) { + return "Identity32"; + } + else if (std::is_same::value) { + return "Identity64"; + } + else { + return "UnrecognizedIdentity"; + } + } + + template + const std::string IdentityOf::location(int64_t where) const { + std::stringstream out; + int64_t fieldi = 0; + int64_t widthi = 0; + for (int64_t bothi = 0; bothi < (int64_t)fieldloc_.size() + width_; bothi++) { + if (bothi != 0) { + out << ", "; + } + if (fieldi < (int64_t)fieldloc_.size() && fieldloc_[fieldi].first == bothi) { + out << "\"" << fieldloc_[fieldi].second << "\""; + fieldi++; + } + else { + out << ptr_.get()[offset_ + where*width_ + widthi]; + widthi++; + } + } + return out.str(); + } + template const std::shared_ptr IdentityOf::to64() const { if (std::is_same::value) { @@ -41,14 +75,14 @@ namespace awkward { else if (std::is_same::value) { name = "Identity64"; } - out << indent << pre << "<" << name << " ref=\"" << ref() << "\" fieldloc=\"["; - for (size_t i = 0; i < fieldloc().size(); i++) { + out << indent << pre << "<" << name << " ref=\"" << ref_ << "\" fieldloc=\"["; + for (size_t i = 0; i < fieldloc_.size(); i++) { if (i != 0) { out << " "; } - out << "(" << fieldloc()[i].first << ", '" << fieldloc()[i].second << "')"; + out << "(" << fieldloc_[i].first << ", '" << fieldloc_[i].second << "')"; } - out << "]\" width=\"" << width() << "\" length=\"" << length() << "\" at=\"0x"; + out << "]\" width=\"" << width_ << "\" offset=\"" << offset_ << "\" length=\"" << length_ << "\" at=\"0x"; out << std::hex << std::setw(12) << std::setfill('0') << reinterpret_cast(ptr_.get()) << "\"/>" << post; return out.str(); } @@ -74,9 +108,8 @@ namespace awkward { IdentityOf* rawout = new IdentityOf(ref_, fieldloc_, width_, carry.length()); std::shared_ptr out(rawout); - Error assign_err = kNoError; if (std::is_same::value) { - assign_err = awkward_identity32_getitem_carry_64( + Error err = awkward_identity32_getitem_carry_64( reinterpret_cast(rawout->ptr().get()), reinterpret_cast(ptr_.get()), carry.ptr().get(), @@ -84,9 +117,10 @@ namespace awkward { offset_, width_, length_); + util::handle_error(err, classname(), nullptr); } else if (std::is_same::value) { - assign_err = awkward_identity64_getitem_carry_64( + Error err = awkward_identity64_getitem_carry_64( reinterpret_cast(rawout->ptr().get()), reinterpret_cast(ptr_.get()), carry.ptr().get(), @@ -94,11 +128,11 @@ namespace awkward { offset_, width_, length_); + util::handle_error(err, classname(), nullptr); } else { throw std::runtime_error("unrecognized Identity specialization"); } - HANDLE_ERROR(assign_err) return out; } diff --git a/src/libawkward/Index.cpp b/src/libawkward/Index.cpp index ffd3a02687..5a1d9fe7d3 100644 --- a/src/libawkward/Index.cpp +++ b/src/libawkward/Index.cpp @@ -51,7 +51,7 @@ namespace awkward { out << (int64_t)getitem_at(i); } } - out << "]\" at=\"0x"; + out << "]\" offset=\"" << offset_ << "\" at=\"0x"; out << std::hex << std::setw(12) << std::setfill('0') << reinterpret_cast(ptr_.get()) << "\"/>" << post; return out.str(); } diff --git a/src/libawkward/ListArray.cpp b/src/libawkward/ListArray.cpp index 60c4d84692..00e1b7ff25 100644 --- a/src/libawkward/ListArray.cpp +++ b/src/libawkward/ListArray.cpp @@ -11,6 +11,19 @@ #include "awkward/ListArray.h" namespace awkward { + template + const std::string ListArrayOf::classname() const { + if (std::is_same::value) { + return "ListArray32"; + } + else if (std::is_same::value) { + return "ListArray64"; + } + else { + return "UnrecognizedListArray"; + } + } + template <> void ListArrayOf::setid(const std::shared_ptr id) { if (id.get() == nullptr) { @@ -38,7 +51,7 @@ namespace awkward { content_.get()->length(), length(), rawid->width()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); content_.get()->setid(subid); } else if (Identity64* rawid = dynamic_cast(bigid.get())) { @@ -55,7 +68,7 @@ namespace awkward { content_.get()->length(), length(), rawid->width()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); content_.get()->setid(subid); } else { @@ -89,7 +102,7 @@ namespace awkward { content_.get()->length(), length(), rawid->width()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); content_.get()->setid(subid); } else { @@ -104,13 +117,15 @@ namespace awkward { if (length() <= kMaxInt32) { Identity32* rawid = new Identity32(Identity::newref(), Identity::FieldLoc(), 1, length()); std::shared_ptr newid(rawid); - awkward_new_identity32(rawid->ptr().get(), length()); + Error err = awkward_new_identity32(rawid->ptr().get(), length()); + util::handle_error(err, classname(), id_.get()); setid(newid); } else { Identity64* rawid = new Identity64(Identity::newref(), Identity::FieldLoc(), 1, length()); std::shared_ptr newid(rawid); - awkward_new_identity64(rawid->ptr().get(), length()); + Error err = awkward_new_identity64(rawid->ptr().get(), length()); + util::handle_error(err, classname(), id_.get()); setid(newid); } } @@ -118,21 +133,14 @@ namespace awkward { template const std::string ListArrayOf::tostring_part(const std::string indent, const std::string pre, const std::string post) const { std::stringstream out; - std::string name = "Unrecognized ListArray"; - if (std::is_same::value) { - name = "ListArray32"; - } - else if (std::is_same::value) { - name = "ListArray64"; - } - out << indent << pre << "<" << name << ">\n"; + out << indent << pre << "<" << classname() << ">\n"; if (id_.get() != nullptr) { out << id_.get()->tostring_part(indent + std::string(" "), "", "\n"); } out << starts_.tostring_part(indent + std::string(" "), "", "\n"); out << stops_.tostring_part(indent + std::string(" "), "", "\n"); out << content_.get()->tostring_part(indent + std::string(" "), "", "\n"); - out << indent << "" << post; + out << indent << "" << post; return out.str(); } @@ -153,10 +161,10 @@ namespace awkward { regular_at += starts_.length(); } if (regular_at < 0 || regular_at >= starts_.length()) { - throw std::invalid_argument("index out of range"); + util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); } if (regular_at >= stops_.length()) { - throw std::invalid_argument("len(stops) < len(starts) in ListArray"); + throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); } return content_.get()->getitem_range(starts_.getitem_at(regular_at), stops_.getitem_at(regular_at)); } @@ -167,13 +175,13 @@ namespace awkward { int64_t regular_stop = stop; awkward_regularize_rangeslice(®ular_start, ®ular_stop, true, start != Slice::none(), stop != Slice::none(), starts_.length()); if (regular_stop > stops_.length()) { - throw std::invalid_argument("len(stops) < len(starts) in ListArray"); + throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); } std::shared_ptr id(nullptr); if (id_.get() != nullptr) { if (regular_stop > id_.get()->length()) { - throw std::invalid_argument("index out of range for identity"); + util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); } id = id_.get()->getitem_range(regular_start, regular_stop); } @@ -185,7 +193,7 @@ namespace awkward { const std::shared_ptr ListArrayOf::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { int64_t lenstarts = starts_.length(); if (stops_.length() < lenstarts) { - throw std::invalid_argument("len(stops) < len(starts)"); + throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); } if (head.get() == nullptr) { @@ -205,6 +213,7 @@ namespace awkward { starts_.offset(), stops_.offset(), at->at()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); return nextcontent.get()->getitem_next(nexthead, nexttail, advanced); } @@ -219,7 +228,7 @@ namespace awkward { step = 1; } int64_t carrylength; - awkward_listarray32_getitem_next_range_carrylength( + Error err1 = awkward_listarray32_getitem_next_range_carrylength( &carrylength, starts_.ptr().get(), stops_.ptr().get(), @@ -229,11 +238,12 @@ namespace awkward { start, stop, step); + util::handle_error(err1, classname(), id_.get()); Index32 nextoffsets(lenstarts + 1); Index64 nextcarry(carrylength); - awkward_listarray32_getitem_next_range_64( + Error err2 = awkward_listarray32_getitem_next_range_64( nextoffsets.ptr().get(), nextcarry.ptr().get(), starts_.ptr().get(), @@ -244,6 +254,7 @@ namespace awkward { start, stop, step); + util::handle_error(err2, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); if (advanced.length() == 0) { @@ -251,16 +262,18 @@ namespace awkward { } else { int64_t total; - awkward_listarray32_getitem_next_range_counts_64( + Error err1 = awkward_listarray32_getitem_next_range_counts_64( &total, nextoffsets.ptr().get(), lenstarts); + util::handle_error(err1, classname(), id_.get()); Index64 nextadvanced(total); - awkward_listarray32_getitem_next_range_spreadadvanced_64( + Error err2 = awkward_listarray32_getitem_next_range_spreadadvanced_64( nextadvanced.ptr().get(), advanced.ptr().get(), nextoffsets.ptr().get(), lenstarts); + util::handle_error(err2, classname(), id_.get()); return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); } } @@ -293,7 +306,7 @@ namespace awkward { lenstarts, flathead.length(), content_.get()->length()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); @@ -313,7 +326,7 @@ namespace awkward { lenstarts, flathead.length(), content_.get()->length()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); return nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced); } @@ -328,7 +341,7 @@ namespace awkward { const std::shared_ptr ListArrayOf::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { int64_t lenstarts = starts_.length(); if (stops_.length() < lenstarts) { - throw std::invalid_argument("len(stops) < len(starts)"); + throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); } if (head.get() == nullptr) { @@ -348,6 +361,7 @@ namespace awkward { starts_.offset(), stops_.offset(), at->at()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); return nextcontent.get()->getitem_next(nexthead, nexttail, advanced); } @@ -362,7 +376,7 @@ namespace awkward { step = 1; } int64_t carrylength; - awkward_listarray64_getitem_next_range_carrylength( + Error err1 = awkward_listarray64_getitem_next_range_carrylength( &carrylength, starts_.ptr().get(), stops_.ptr().get(), @@ -372,11 +386,12 @@ namespace awkward { start, stop, step); + util::handle_error(err1, classname(), id_.get()); Index64 nextoffsets(lenstarts + 1); Index64 nextcarry(carrylength); - awkward_listarray64_getitem_next_range_64( + Error err2 = awkward_listarray64_getitem_next_range_64( nextoffsets.ptr().get(), nextcarry.ptr().get(), starts_.ptr().get(), @@ -387,6 +402,7 @@ namespace awkward { start, stop, step); + util::handle_error(err2, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); if (advanced.length() == 0) { @@ -394,16 +410,18 @@ namespace awkward { } else { int64_t total; - awkward_listarray64_getitem_next_range_counts_64( + Error err1 = awkward_listarray64_getitem_next_range_counts_64( &total, nextoffsets.ptr().get(), lenstarts); + util::handle_error(err1, classname(), id_.get()); Index64 nextadvanced(total); - awkward_listarray64_getitem_next_range_spreadadvanced_64( + Error err2 = awkward_listarray64_getitem_next_range_spreadadvanced_64( nextadvanced.ptr().get(), advanced.ptr().get(), nextoffsets.ptr().get(), lenstarts); + util::handle_error(err2, classname(), id_.get()); return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); } } @@ -436,7 +454,7 @@ namespace awkward { lenstarts, flathead.length(), content_.get()->length()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); @@ -456,7 +474,7 @@ namespace awkward { lenstarts, flathead.length(), content_.get()->length()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); return nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced); } @@ -469,12 +487,13 @@ namespace awkward { template <> const std::shared_ptr ListArrayOf::carry(const Index64& carry) const { - if (stops_.length() < starts_.length()) { - throw std::invalid_argument("len(stops) < len(starts)"); + int64_t lenstarts = starts_.length(); + if (stops_.length() < lenstarts) { + throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); } Index32 nextstarts(carry.length()); Index32 nextstops(carry.length()); - awkward_listarray32_getitem_carry_64( + Error err = awkward_listarray32_getitem_carry_64( nextstarts.ptr().get(), nextstops.ptr().get(), starts_.ptr().get(), @@ -482,7 +501,9 @@ namespace awkward { carry.ptr().get(), starts_.offset(), stops_.offset(), + lenstarts, carry.length()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr id(nullptr); if (id_.get() != nullptr) { id = id_.get()->getitem_carry_64(carry); @@ -492,12 +513,13 @@ namespace awkward { template <> const std::shared_ptr ListArrayOf::carry(const Index64& carry) const { - if (stops_.length() < starts_.length()) { - throw std::invalid_argument("len(stops) < len(starts)"); + int64_t lenstarts = starts_.length(); + if (stops_.length() < lenstarts) { + throw std::invalid_argument(std::string("in ") + classname() + std::string(", len(stops) < len(starts)")); } Index64 nextstarts(carry.length()); Index64 nextstops(carry.length()); - awkward_listarray64_getitem_carry_64( + Error err = awkward_listarray64_getitem_carry_64( nextstarts.ptr().get(), nextstops.ptr().get(), starts_.ptr().get(), @@ -505,7 +527,9 @@ namespace awkward { carry.ptr().get(), starts_.offset(), stops_.offset(), + lenstarts, carry.length()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr id(nullptr); if (id_.get() != nullptr) { id = id_.get()->getitem_carry_64(carry); diff --git a/src/libawkward/ListOffsetArray.cpp b/src/libawkward/ListOffsetArray.cpp index 4e85e4d60b..62a165d037 100644 --- a/src/libawkward/ListOffsetArray.cpp +++ b/src/libawkward/ListOffsetArray.cpp @@ -9,6 +9,19 @@ #include "awkward/ListOffsetArray.h" namespace awkward { + template + const std::string ListOffsetArrayOf::classname() const { + if (std::is_same::value) { + return "ListOffsetArray32"; + } + else if (std::is_same::value) { + return "ListOffsetArray64"; + } + else { + return "UnrecognizedListOffsetArray"; + } + } + template IndexOf make_starts(const IndexOf& offsets) { return IndexOf(offsets.ptr(), offsets.offset(), offsets.length() - 1); @@ -48,7 +61,7 @@ namespace awkward { content_.get()->length(), length(), rawid->width()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); content_.get()->setid(subid); } else if (Identity64* rawid = dynamic_cast(bigid.get())) { @@ -65,7 +78,7 @@ namespace awkward { content_.get()->length(), length(), rawid->width()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); content_.get()->setid(subid); } else { @@ -101,7 +114,7 @@ namespace awkward { content_.get()->length(), length(), rawid->width()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); content_.get()->setid(subid); } else { @@ -116,13 +129,15 @@ namespace awkward { if (length() <= kMaxInt32) { Identity32* rawid = new Identity32(Identity::newref(), Identity::FieldLoc(), 1, length()); std::shared_ptr newid(rawid); - awkward_new_identity32(rawid->ptr().get(), length()); + Error err = awkward_new_identity32(rawid->ptr().get(), length()); + util::handle_error(err, classname(), id_.get()); setid(newid); } else { Identity64* rawid = new Identity64(Identity::newref(), Identity::FieldLoc(), 1, length()); std::shared_ptr newid(rawid); - awkward_new_identity64(rawid->ptr().get(), length()); + Error err = awkward_new_identity64(rawid->ptr().get(), length()); + util::handle_error(err, classname(), id_.get()); setid(newid); } } @@ -130,20 +145,13 @@ namespace awkward { template const std::string ListOffsetArrayOf::tostring_part(const std::string indent, const std::string pre, const std::string post) const { std::stringstream out; - std::string name = "Unrecognized ListOffsetArray"; - if (std::is_same::value) { - name = "ListOffsetArray32"; - } - else if (std::is_same::value) { - name = "ListOffsetArray64"; - } - out << indent << pre << "<" << name << ">\n"; + out << indent << pre << "<" << classname() << ">\n"; if (id_.get() != nullptr) { out << id_.get()->tostring_part(indent + std::string(" "), "", "\n"); } out << offsets_.tostring_part(indent + std::string(" "), "", "\n"); out << content_.get()->tostring_part(indent + std::string(" "), "", "\n"); - out << indent << "" << post; + out << indent << "" << post; return out.str(); } @@ -173,7 +181,7 @@ namespace awkward { std::shared_ptr id(nullptr); if (id_.get() != nullptr) { if (regular_stop > id_.get()->length()) { - throw std::invalid_argument("index out of range for identity"); + util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); } id = id_.get()->getitem_range(regular_start, regular_stop); } @@ -204,6 +212,7 @@ namespace awkward { starts.offset(), stops.offset(), at->at()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); return nextcontent.get()->getitem_next(nexthead, nexttail, advanced); } @@ -220,7 +229,7 @@ namespace awkward { step = 1; } int64_t carrylength; - awkward_listarray32_getitem_next_range_carrylength( + Error err1 = awkward_listarray32_getitem_next_range_carrylength( &carrylength, starts.ptr().get(), stops.ptr().get(), @@ -230,11 +239,12 @@ namespace awkward { start, stop, step); + util::handle_error(err1, classname(), id_.get()); Index32 nextoffsets(lenstarts + 1); Index64 nextcarry(carrylength); - awkward_listarray32_getitem_next_range_64( + Error err2 = awkward_listarray32_getitem_next_range_64( nextoffsets.ptr().get(), nextcarry.ptr().get(), starts.ptr().get(), @@ -245,6 +255,7 @@ namespace awkward { start, stop, step); + util::handle_error(err2, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); if (advanced.length() == 0) { @@ -252,16 +263,18 @@ namespace awkward { } else { int64_t total; - awkward_listarray32_getitem_next_range_counts_64( + Error err1 = awkward_listarray32_getitem_next_range_counts_64( &total, nextoffsets.ptr().get(), lenstarts); + util::handle_error(err1, classname(), id_.get()); Index64 nextadvanced(total); - awkward_listarray32_getitem_next_range_spreadadvanced_64( + Error err2 = awkward_listarray32_getitem_next_range_spreadadvanced_64( nextadvanced.ptr().get(), advanced.ptr().get(), nextoffsets.ptr().get(), lenstarts); + util::handle_error(err2, classname(), id_.get()); return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); } } @@ -297,7 +310,7 @@ namespace awkward { lenstarts, flathead.length(), content_.get()->length()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); @@ -317,7 +330,7 @@ namespace awkward { lenstarts, flathead.length(), content_.get()->length()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); return nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced); } @@ -351,6 +364,7 @@ namespace awkward { starts.offset(), stops.offset(), at->at()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); return nextcontent.get()->getitem_next(nexthead, nexttail, advanced); } @@ -367,7 +381,7 @@ namespace awkward { step = 1; } int64_t carrylength; - awkward_listarray64_getitem_next_range_carrylength( + Error err1 = awkward_listarray64_getitem_next_range_carrylength( &carrylength, starts.ptr().get(), stops.ptr().get(), @@ -377,11 +391,12 @@ namespace awkward { start, stop, step); + util::handle_error(err1, classname(), id_.get()); Index64 nextoffsets(lenstarts + 1); Index64 nextcarry(carrylength); - awkward_listarray64_getitem_next_range_64( + Error err2 = awkward_listarray64_getitem_next_range_64( nextoffsets.ptr().get(), nextcarry.ptr().get(), starts.ptr().get(), @@ -392,6 +407,7 @@ namespace awkward { start, stop, step); + util::handle_error(err2, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); if (advanced.length() == 0) { @@ -399,16 +415,18 @@ namespace awkward { } else { int64_t total; - awkward_listarray64_getitem_next_range_counts_64( + Error err1 = awkward_listarray64_getitem_next_range_counts_64( &total, nextoffsets.ptr().get(), lenstarts); + util::handle_error(err1, classname(), id_.get()); Index64 nextadvanced(total); - awkward_listarray64_getitem_next_range_spreadadvanced_64( + Error err2 = awkward_listarray64_getitem_next_range_spreadadvanced_64( nextadvanced.ptr().get(), advanced.ptr().get(), nextoffsets.ptr().get(), lenstarts); + util::handle_error(err2, classname(), id_.get()); return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); } } @@ -444,7 +462,7 @@ namespace awkward { lenstarts, flathead.length(), content_.get()->length()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); // FIXME: if the head is not flat, you'll need to wrap the ListArray output in a RegularArray return std::shared_ptr(new ListOffsetArrayOf(id_, nextoffsets, nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced))); @@ -464,7 +482,7 @@ namespace awkward { lenstarts, flathead.length(), content_.get()->length()); - HANDLE_ERROR(err) + util::handle_error(err, classname(), id_.get()); std::shared_ptr nextcontent = content_.get()->carry(nextcarry); return nextcontent.get()->getitem_next(nexthead, nexttail, nextadvanced); } @@ -481,7 +499,7 @@ namespace awkward { Index32 stops = make_stops(offsets_); Index32 nextstarts(carry.length()); Index32 nextstops(carry.length()); - awkward_listarray32_getitem_carry_64( + Error err = awkward_listarray32_getitem_carry_64( nextstarts.ptr().get(), nextstops.ptr().get(), starts.ptr().get(), @@ -489,7 +507,9 @@ namespace awkward { carry.ptr().get(), starts.offset(), stops.offset(), + offsets_.length() - 1, carry.length()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr id(nullptr); if (id_.get() != nullptr) { id = id_.get()->getitem_carry_64(carry); @@ -503,7 +523,7 @@ namespace awkward { Index64 stops = make_stops(offsets_); Index64 nextstarts(carry.length()); Index64 nextstops(carry.length()); - awkward_listarray64_getitem_carry_64( + Error err = awkward_listarray64_getitem_carry_64( nextstarts.ptr().get(), nextstops.ptr().get(), starts.ptr().get(), @@ -511,7 +531,9 @@ namespace awkward { carry.ptr().get(), starts.offset(), stops.offset(), + offsets_.length() - 1, carry.length()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr id(nullptr); if (id_.get() != nullptr) { id = id_.get()->getitem_carry_64(carry); diff --git a/src/libawkward/NumpyArray.cpp b/src/libawkward/NumpyArray.cpp index 7cf40e4503..b34a4f880e 100644 --- a/src/libawkward/NumpyArray.cpp +++ b/src/libawkward/NumpyArray.cpp @@ -42,9 +42,11 @@ namespace awkward { return *reinterpret_cast(reinterpret_cast(ptr_.get()) + byteoffset_ + at); } + const std::string NumpyArray::classname() const { return "NumpyArray"; } + void NumpyArray::setid(const std::shared_ptr id) { if (id.get() != nullptr && length() != id.get()->length()) { - throw std::invalid_argument("content and its id must have the same length"); + util::handle_error(failure("content and its id must have the same length", kSliceNone, kSliceNone), classname(), id_.get()); } id_ = id; } @@ -54,13 +56,15 @@ namespace awkward { if (length() <= kMaxInt32) { Identity32* rawid = new Identity32(Identity::newref(), Identity::FieldLoc(), 1, length()); std::shared_ptr newid(rawid); - awkward_new_identity32(rawid->ptr().get(), length()); + Error err = awkward_new_identity32(rawid->ptr().get(), length()); + util::handle_error(err, classname(), id_.get()); setid(newid); } else { Identity64* rawid = new Identity64(Identity::newref(), Identity::FieldLoc(), 1, length()); std::shared_ptr newid(rawid); - awkward_new_identity64(rawid->ptr().get(), length()); + Error err = awkward_new_identity64(rawid->ptr().get(), length()); + util::handle_error(err, classname(), id_.get()); setid(newid); } } @@ -95,7 +99,7 @@ namespace awkward { const std::string NumpyArray::tostring_part(const std::string indent, const std::string pre, const std::string post) const { assert(!isscalar()); std::stringstream out; - out << indent << pre << "\n"; out << id_.get()->tostring_part(indent + std::string(" "), "", "\n"); - out << indent << "" << post; + out << indent << "" << post; } return out.str(); } @@ -193,7 +197,7 @@ namespace awkward { regular_at += shape_[0]; } if (regular_at < 0 || regular_at >= shape_[0]) { - throw std::invalid_argument("index out of range"); + util::handle_error(failure("index out of range", kSliceNone, at), classname(), id_.get()); } ssize_t byteoffset = byteoffset_ + strides_[0]*((ssize_t)regular_at); const std::vector shape(shape_.begin() + 1, shape_.end()); @@ -201,7 +205,7 @@ namespace awkward { std::shared_ptr id; if (id_.get() != nullptr) { if (regular_at >= id_.get()->length()) { - throw std::invalid_argument("index out of range for identity"); + util::handle_error(failure("index out of range", kSliceNone, at), id_.get()->classname(), nullptr); } id = id_.get()->getitem_range(regular_at, regular_at + 1); } @@ -220,7 +224,7 @@ namespace awkward { std::shared_ptr id; if (id_.get() != nullptr) { if (regular_stop > id_.get()->length()) { - throw std::invalid_argument("index out of range for identity"); + util::handle_error(failure("index out of range", kSliceNone, stop), id_.get()->classname(), nullptr); } id = id_.get()->getitem_range(regular_start, regular_stop); } @@ -271,7 +275,8 @@ namespace awkward { const std::shared_ptr NumpyArray::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& advanced) const { assert(!isscalar()); Index64 carry(shape_[0]); - awkward_carry_arange_64(carry.ptr().get(), shape_[0]); + Error err = awkward_carry_arange_64(carry.ptr().get(), shape_[0]); + util::handle_error(err, classname(), id_.get()); return getitem_next(head, tail, carry, advanced, shape_[0], strides_[0]).shallow_copy(); } @@ -279,13 +284,14 @@ namespace awkward { assert(!isscalar()); std::shared_ptr ptr(new uint8_t[(size_t)(carry.length()*strides_[0])], awkward::util::array_deleter()); - awkward_numpyarray_getitem_next_null_64( + Error err = awkward_numpyarray_getitem_next_null_64( reinterpret_cast(ptr.get()), reinterpret_cast(ptr_.get()), carry.length(), strides_[0], byteoffset_, carry.ptr().get()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr id(nullptr); if (id_.get() != nullptr) { @@ -347,7 +353,8 @@ namespace awkward { } else { Index64 bytepos(shape_[0]); - awkward_numpyarray_contiguous_init_64(bytepos.ptr().get(), shape_[0], strides_[0]); + Error err = awkward_numpyarray_contiguous_init_64(bytepos.ptr().get(), shape_[0], strides_[0]); + util::handle_error(err, classname(), id_.get()); return contiguous_next(bytepos); } } @@ -355,25 +362,27 @@ namespace awkward { const NumpyArray NumpyArray::contiguous_next(Index64 bytepos) const { if (iscontiguous()) { std::shared_ptr ptr(new uint8_t[(size_t)(bytepos.length()*strides_[0])], awkward::util::array_deleter()); - awkward_numpyarray_contiguous_copy_64( + Error err = awkward_numpyarray_contiguous_copy_64( reinterpret_cast(ptr.get()), reinterpret_cast(ptr_.get()), bytepos.length(), strides_[0], byteoffset_, bytepos.ptr().get()); + util::handle_error(err, classname(), id_.get()); return NumpyArray(id_, ptr, shape_, strides_, 0, itemsize_, format_); } else if (shape_.size() == 1) { std::shared_ptr ptr(new uint8_t[(size_t)(bytepos.length()*itemsize_)], awkward::util::array_deleter()); - awkward_numpyarray_contiguous_copy_64( + Error err = awkward_numpyarray_contiguous_copy_64( reinterpret_cast(ptr.get()), reinterpret_cast(ptr_.get()), bytepos.length(), itemsize_, byteoffset_, bytepos.ptr().get()); + util::handle_error(err, classname(), id_.get()); std::vector strides = { itemsize_ }; return NumpyArray(id_, ptr, shape_, strides, 0, itemsize_, format_); } @@ -382,12 +391,13 @@ namespace awkward { NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); Index64 nextbytepos(bytepos.length()*shape_[1]); - awkward_numpyarray_contiguous_next_64( + Error err = awkward_numpyarray_contiguous_next_64( nextbytepos.ptr().get(), bytepos.ptr().get(), bytepos.length(), (int64_t)shape_[1], (int64_t)strides_[1]); + util::handle_error(err, classname(), id_.get()); NumpyArray out = next.contiguous_next(nextbytepos); std::vector outstrides = { shape_[1]*out.strides_[0] }; @@ -403,13 +413,13 @@ namespace awkward { else if (SliceAt* at = dynamic_cast(head.get())) { if (ndim() < 2) { - throw std::invalid_argument("too many indexes for array"); + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); } int64_t i = at->at(); if (i < 0) i += shape_[1]; if (i < 0 || i >= shape_[1]) { - throw std::invalid_argument("index out of range"); + util::handle_error(failure("index out of range", kSliceNone, at->at()), classname(), id_.get()); } ssize_t nextbyteoffset = byteoffset_ + ((ssize_t)i)*strides_[1]; @@ -426,7 +436,7 @@ namespace awkward { else if (SliceRange* range = dynamic_cast(head.get())) { if (ndim() < 2) { - throw std::invalid_argument("too many indexes for array"); + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); } int64_t start = range->start(); @@ -473,7 +483,7 @@ namespace awkward { items.insert(items.end(), tailitems.begin(), tailitems.end()); std::shared_ptr nexthead(new SliceRange(Slice::none(), Slice::none(), 1)); - Slice nexttail(items, true); + Slice nexttail(items); return getitem_bystrides(nexthead, nexttail, length); } } @@ -498,13 +508,14 @@ namespace awkward { const NumpyArray NumpyArray::getitem_next(const std::shared_ptr head, const Slice& tail, const Index64& carry, const Index64& advanced, int64_t length, int64_t stride) const { if (head.get() == nullptr) { std::shared_ptr ptr(new uint8_t[(size_t)(carry.length()*stride)], awkward::util::array_deleter()); - awkward_numpyarray_getitem_next_null_64( + Error err = awkward_numpyarray_getitem_next_null_64( reinterpret_cast(ptr.get()), reinterpret_cast(ptr_.get()), carry.length(), stride, byteoffset_, carry.ptr().get()); + util::handle_error(err, classname(), id_.get()); std::shared_ptr id(nullptr); if (id_.get() != nullptr) { @@ -520,7 +531,7 @@ namespace awkward { else if (SliceAt* at = dynamic_cast(head.get())) { if (ndim() < 2) { - throw std::invalid_argument("too many indexes for array"); + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); } NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); @@ -530,13 +541,22 @@ namespace awkward { // if we had any array slices, this int would become an array assert(advanced.length() == 0); + int64_t regular_at = at->at(); + if (regular_at < 0) { + regular_at += shape_[1]; + } + if (!(0 <= regular_at && regular_at < shape_[1])) { + util::handle_error(failure("index out of range", kSliceNone, at->at()), classname(), id_.get()); + } + Index64 nextcarry(carry.length()); - awkward_numpyarray_getitem_next_at_64( + Error err = awkward_numpyarray_getitem_next_at_64( nextcarry.ptr().get(), carry.ptr().get(), carry.length(), shape_[1], // because this is contiguous - at->at()); + regular_at); + util::handle_error(err, classname(), id_.get()); NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length, next.strides_[0]); @@ -547,7 +567,7 @@ namespace awkward { else if (SliceRange* range = dynamic_cast(head.get())) { if (ndim() < 2) { - throw std::invalid_argument("too many indexes for array"); + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); } int64_t start = range->start(); @@ -570,7 +590,7 @@ namespace awkward { if (advanced.length() == 0) { Index64 nextcarry(carry.length()*lenhead); - awkward_numpyarray_getitem_next_range_64( + Error err = awkward_numpyarray_getitem_next_range_64( nextcarry.ptr().get(), carry.ptr().get(), carry.length(), @@ -578,6 +598,7 @@ namespace awkward { shape_[1], // because this is contiguous start, step); + util::handle_error(err, classname(), id_.get()); NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*lenhead, next.strides_[0]); std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; @@ -590,7 +611,7 @@ namespace awkward { else { Index64 nextcarry(carry.length()*lenhead); Index64 nextadvanced(carry.length()*lenhead); - awkward_numpyarray_getitem_next_range_advanced_64( + Error err = awkward_numpyarray_getitem_next_range_advanced_64( nextcarry.ptr().get(), nextadvanced.ptr().get(), carry.ptr().get(), @@ -600,6 +621,7 @@ namespace awkward { shape_[1], // because this is contiguous start, step); + util::handle_error(err, classname(), id_.get()); NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*lenhead, next.strides_[0]); std::vector outshape = { (ssize_t)length, (ssize_t)lenhead }; @@ -625,7 +647,7 @@ namespace awkward { std::vector> items = { std::shared_ptr(new SliceEllipsis()) }; items.insert(items.end(), tailitems.begin(), tailitems.end()); std::shared_ptr nexthead(new SliceRange(Slice::none(), Slice::none(), 1)); - Slice nexttail(items, true); + Slice nexttail(items); return getitem_next(nexthead, nexttail, carry, advanced, length, stride); } } @@ -644,7 +666,7 @@ namespace awkward { else if (SliceArray64* array = dynamic_cast(head.get())) { if (ndim() < 2) { - throw std::invalid_argument("too many indexes for array"); + util::handle_error(failure("too many dimensions in slice", kSliceNone, kSliceNone), classname(), id_.get()); } NumpyArray next(id_, ptr_, flatten_shape(shape_), flatten_strides(strides_), byteoffset_, itemsize_, format_); @@ -652,16 +674,16 @@ namespace awkward { Slice nexttail = tail.tail(); Index64 flathead = array->ravel(); - Error regularize_error = awkward_regularize_arrayslice_64( + Error err = awkward_regularize_arrayslice_64( flathead.ptr().get(), flathead.length(), shape_[1]); - HANDLE_ERROR(regularize_error) + util::handle_error(err, classname(), id_.get()); if (advanced.length() == 0) { Index64 nextcarry(carry.length()*flathead.length()); Index64 nextadvanced(carry.length()*flathead.length()); - awkward_numpyarray_getitem_next_array_64( + Error err = awkward_numpyarray_getitem_next_array_64( nextcarry.ptr().get(), nextadvanced.ptr().get(), carry.ptr().get(), @@ -669,6 +691,7 @@ namespace awkward { carry.length(), flathead.length(), shape_[1]); // because this is contiguous + util::handle_error(err, classname(), id_.get()); NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, nextadvanced, length*flathead.length(), next.strides_[0]); @@ -689,13 +712,14 @@ namespace awkward { else { Index64 nextcarry(carry.length()); Index64 nextadvanced(carry.length()); - awkward_numpyarray_getitem_next_array_advanced_64( + Error err = awkward_numpyarray_getitem_next_array_advanced_64( nextcarry.ptr().get(), carry.ptr().get(), advanced.ptr().get(), flathead.ptr().get(), carry.length(), shape_[1]); // because this is contiguous + util::handle_error(err, classname(), id_.get()); NumpyArray out = next.getitem_next(nexthead, nexttail, nextcarry, advanced, length*array->length(), next.strides_[0]); diff --git a/src/libawkward/util.cpp b/src/libawkward/util.cpp new file mode 100644 index 0000000000..6fe5fa813b --- /dev/null +++ b/src/libawkward/util.cpp @@ -0,0 +1,32 @@ +// BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +#include +#include + +#include "awkward/util.h" +#include "awkward/Identity.h" + +namespace awkward { + namespace util { + void handle_error(const Error& err, const std::string classname, const Identity* id) { + if (err.str != nullptr) { + std::stringstream out; + out << "in " << classname; + if (err.location != kSliceNone && id != nullptr) { + assert(err.location > 0); + if (0 <= err.location && err.location < id->length()) { + out << " at id[" << id->location(err.location) << "]"; + } + else { + out << " at id[???]"; + } + } + if (err.attempt != kSliceNone) { + out << " attempting to get " << err.attempt; + } + out << ", " << err.str; + throw std::invalid_argument(out.str()); + } + } + } +} diff --git a/tests/test_PR12_listarray_in_numba.py b/tests/test_PR12_listarray_in_numba.py index 305716b001..766ff3d0d0 100644 --- a/tests/test_PR12_listarray_in_numba.py +++ b/tests/test_PR12_listarray_in_numba.py @@ -65,7 +65,7 @@ def f3(q): def test_array(): @numba.njit - def f4(q): + def f1(q): return q[[2, 0, 0, 1],] - assert awkward1.tolist(f4(array)) == [[3.3, 4.4], [0.0, 1.1, 2.2], [0.0, 1.1, 2.2], []] + assert awkward1.tolist(f1(array)) == [[3.3, 4.4], [0.0, 1.1, 2.2], [0.0, 1.1, 2.2], []] diff --git a/tests/test_PR13_error_handling_struct.py b/tests/test_PR13_error_handling_struct.py new file mode 100644 index 0000000000..01f8882dc3 --- /dev/null +++ b/tests/test_PR13_error_handling_struct.py @@ -0,0 +1,269 @@ +# BSD 3-Clause License; see https://github.com/jpivarski/awkward-1.0/blob/master/LICENSE + +import sys + +import pytest +import numpy +numba = pytest.importorskip("numba") + +import awkward1 +awkward1_numba_util = pytest.importorskip("awkward1._numba.util") + +py27 = (sys.version_info[0] < 3) + +def test_numpyarray(): + array = awkward1.layout.NumpyArray(numpy.arange(10)*1.1) + + with pytest.raises(ValueError) as excinfo: + array[20] + assert str(excinfo.value) == "in NumpyArray attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[-20] + assert str(excinfo.value) == "in NumpyArray attempting to get -20, index out of range" + + array[-20:20] + + with pytest.raises(ValueError) as excinfo: + array[20,] + assert str(excinfo.value) == "in NumpyArray attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[-20,] + assert str(excinfo.value) == "in NumpyArray attempting to get -20, index out of range" + + array[-20:20,] + + with pytest.raises(ValueError) as excinfo: + array[2, 3] + assert str(excinfo.value) == "in NumpyArray, too many dimensions in slice" + + with pytest.raises(ValueError) as excinfo: + array[[5, 3, 20, 8]] + assert str(excinfo.value) == "in NumpyArray attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[[5, 3, -20, 8]] + assert str(excinfo.value) == "in NumpyArray attempting to get -20, index out of range" + + array.setid() + + with pytest.raises(ValueError) as excinfo: + array[20] + assert str(excinfo.value) == "in NumpyArray attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[-20] + assert str(excinfo.value) == "in NumpyArray attempting to get -20, index out of range" + + array[-20:20] + + with pytest.raises(ValueError) as excinfo: + array[20,] + assert str(excinfo.value) == "in NumpyArray attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[-20,] + assert str(excinfo.value) == "in NumpyArray attempting to get -20, index out of range" + + array[-20:20,] + + with pytest.raises(ValueError) as excinfo: + array[2, 3] + assert str(excinfo.value) == "in NumpyArray, too many dimensions in slice" + + with pytest.raises(ValueError) as excinfo: + array[[5, 3, 20, 8]] + assert str(excinfo.value) == "in NumpyArray attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[[5, 3, -20, 8]] + assert str(excinfo.value) == "in NumpyArray attempting to get -20, index out of range" + +def test_listarray_numpyarray(): + starts = awkward1.layout.Index64(numpy.array([0, 3, 3, 5, 6])) + stops = awkward1.layout.Index64(numpy.array([3, 3, 5, 6])) + content = awkward1.layout.NumpyArray(numpy.arange(10)*1.1) + array = awkward1.layout.ListArray64(starts, stops, content) + + with pytest.raises(ValueError) as excinfo: + array[4] + assert str(excinfo.value) == "in ListArray64, len(stops) < len(starts)" + + starts = awkward1.layout.Index64(numpy.array([0, 3, 3, 5, 6])) + stops = awkward1.layout.Index64(numpy.array([3, 3, 5, 6, 10])) + content = awkward1.layout.NumpyArray(numpy.arange(10)*1.1) + array = awkward1.layout.ListArray64(starts, stops, content) + + with pytest.raises(ValueError) as excinfo: + array[20] + assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[-20] + assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + + array[-20:20] + + with pytest.raises(ValueError) as excinfo: + array[20,] + assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[-20,] + assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + + array[-20:20,] + + with pytest.raises(ValueError) as excinfo: + array[2, 1, 0] + assert str(excinfo.value) == "in NumpyArray, too many dimensions in slice" + + with pytest.raises(ValueError) as excinfo: + array[[2, 0, 0, 20, 3]] + assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[[2, 0, 0, -20, 3]] + assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + + starts = awkward1.layout.Index64(numpy.array([0, 3, 3, 5, 6])) + stops = awkward1.layout.Index64(numpy.array([3, 3, 5, 6, 10])) + content = awkward1.layout.NumpyArray(numpy.arange(10)*1.1) + array = awkward1.layout.ListArray64(starts, stops, content) + + array.setid() + + with pytest.raises(ValueError) as excinfo: + array[2, 20] + assert str(excinfo.value) == "in ListArray64 at id[2] attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[2, -20] + assert str(excinfo.value) == "in ListArray64 at id[2] attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[1:][2, 20] + assert str(excinfo.value) == "in ListArray64 at id[3] attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[1:][2, -20] + assert str(excinfo.value) == "in ListArray64 at id[3] attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[2, [1, 0, 0, 20]] + assert str(excinfo.value) == "in ListArray64 at id[2] attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[2, [1, 0, 0, -20]] + assert str(excinfo.value) == "in ListArray64 at id[2] attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[1:][2, [0, 20]] + assert str(excinfo.value) == "in ListArray64 at id[3] attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array[1:][2, [0, -20]] + assert str(excinfo.value) == "in ListArray64 at id[3] attempting to get -20, index out of range" + +def test_listarray_listarray_numpyarray(): + content = awkward1.layout.NumpyArray(numpy.array([1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9])) + starts1 = awkward1.layout.Index64(numpy.array([0, 3, 3, 5, 6])) + stops1 = awkward1.layout.Index64(numpy.array([3, 3, 5, 6, 9])) + offsets1 = awkward1.layout.Index64(numpy.array([0, 3, 3, 5, 6, 9])) + starts2 = awkward1.layout.Index64(numpy.array([0, 2, 3, 3])) + stops2 = awkward1.layout.Index64(numpy.array([2, 3, 3, 5])) + offsets2 = awkward1.layout.Index64(numpy.array([0, 2, 3, 3, 5])) + + array1 = awkward1.layout.ListArray64(starts1, stops1, content) + array2 = awkward1.layout.ListArray64(starts2, stops2, array1) + + with pytest.raises(ValueError) as excinfo: + array2[20] + assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[20,] + assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[2, 20] + assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[-20] + assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[-20,] + assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[2, -20] + assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[1, 0, 20] + assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + + array2.setid() + + with pytest.raises(ValueError) as excinfo: + array2[20] + assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[20,] + assert str(excinfo.value) == "in ListArray64 attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[2, 20] + assert str(excinfo.value) == "in ListArray64 at id[2] attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[1:][2, 20] + assert str(excinfo.value) == "in ListArray64 at id[3] attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[-20] + assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[-20,] + assert str(excinfo.value) == "in ListArray64 attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[2, -20] + assert str(excinfo.value) == "in ListArray64 at id[2] attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[1:][2, -20] + assert str(excinfo.value) == "in ListArray64 at id[3] attempting to get -20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[1, 0, 20] + assert str(excinfo.value) == "in ListArray64 at id[1, 0] attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[1:][2, 0, 20] + assert str(excinfo.value) == "in ListArray64 at id[3, 0] attempting to get 20, index out of range" + + with pytest.raises(ValueError) as excinfo: + array2[:, 1:][3, 0, 20] + assert str(excinfo.value) == "in ListArray64 at id[3, 1] attempting to get 20, index out of range" + +def test_array(): + starts = numpy.array([0, 3, 3, 5, 6]) + stops = numpy.array([3, 3, 5, 6, 10]) + content = numpy.array([0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9]) + array = awkward1.layout.ListArray64(awkward1.layout.Index64(starts), + awkward1.layout.Index64(stops), + awkward1.layout.NumpyArray(content)) + + @numba.njit + def f1(q): + return q[[2, 0, 20, 1],] + + with pytest.raises(ValueError) as excinfo: + f1(array) + assert str(excinfo.value) == "in ListArray64, indexing error"