Skip to content

Commit

Permalink
Remove abstraction layer when accessing the 4D code from the DB cursor
Browse files Browse the repository at this point in the history 
Add close statement command when doing execute many
  • Loading branch information
Israel Brewster committed Dec 17, 2014
1 parent 8b10c38 commit 4bb392c
Show file tree
Hide file tree
Showing 4 changed files with 57 additions and 53 deletions.
2 changes: 2 additions & 0 deletions README
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ NOTE: Due to an apparent bug in the C code provided by 4D, or the 4D server itse

v0.3 2014-12-17:
- improve performance of module when receiving large datasets containing date/time values.
- Close statement after execution when doing an insert many
- Fix potential issue when doing large "execute many" queries

v0.2 2014-12-16:
- Fixed lib4d_sql bug preventing proper paging for some queries
Expand Down
105 changes: 53 additions & 52 deletions p4d/p4d.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ def __init__(self, connection, fourdconn, lib4d):

self.fourdconn = fourdconn
self.connection = connection
lib4d_sql = lib4d
self.lib4d_sql = lib4d._lib #so we can address it directly

#----------------------------------------------------------------------
def close(self):
Expand Down Expand Up @@ -239,21 +239,21 @@ def execute(self, query, params=[], describe=True):
break

if self.__prepared == False: #Should always be false, unless we are running an executemany
self.fourd_query = lib4d_sql.fourd_prepare_statement(self.fourdconn, query)
self.fourd_query = self.lib4d_sql.fourd_prepare_statement(self.fourdconn, query)

if self.fourd_query == ffi.NULL:
error = ffi.string(lib4d_sql.fourd_error(self.fourdconn))
error = ffi.string(self.lib4d_sql.fourd_error(self.fourdconn))
raise ProgrammingError(error)

# Some data types need special handling, but most we can just convert to a string.
# All strings need UTF-16LE encoding.
fourdtypes = defaultdict(lambda:lib4d_sql.VK_STRING,
{str: lib4d_sql.VK_STRING,
unicode: lib4d_sql.VK_STRING,
bool: lib4d_sql.VK_BOOLEAN,
int: lib4d_sql.VK_LONG,
long: lib4d_sql.VK_LONG,
float: lib4d_sql.VK_REAL,
fourdtypes = defaultdict(lambda:self.lib4d_sql.VK_STRING,
{str: self.lib4d_sql.VK_STRING,
unicode: self.lib4d_sql.VK_STRING,
bool: self.lib4d_sql.VK_BOOLEAN,
int: self.lib4d_sql.VK_LONG,
long: self.lib4d_sql.VK_LONG,
float: self.lib4d_sql.VK_REAL,
})

for idx, parameter in enumerate(params):
Expand Down Expand Up @@ -287,21 +287,21 @@ def execute(self, query, params=[], describe=True):
param.data = ffi.new("char[]", itemstr.encode('UTF-16LE'))


bound = lib4d_sql.fourd_bind_param(self.fourd_query, idx, fourd_type, param)
bound = self.lib4d_sql.fourd_bind_param(self.fourd_query, idx, fourd_type, param)
if bound != 0:
raise ProgrammingError(ffi.string(lib4d_sql.fourd_error(self.fourdconn)))
raise ProgrammingError(ffi.string(self.lib4d_sql.fourd_error(self.fourdconn)))

# Run the query and return the results
self.result = lib4d_sql.fourd_exec_statement(self.fourd_query, self.pagesize)
self.result = self.lib4d_sql.fourd_exec_statement(self.fourd_query, self.pagesize)

if self.result == ffi.NULL:
raise ProgrammingError(ffi.string(lib4d_sql.fourd_error(self.fourdconn)))
raise ProgrammingError(ffi.string(self.lib4d_sql.fourd_error(self.fourdconn)))

self.__resulttype = self.result.resultType
if self.__resulttype == lib4d_sql.RESULT_SET:
self.__rowcount = lib4d_sql.fourd_num_rows(self.result)
elif self.__resulttype == lib4d_sql.UPDATE_COUNT:
self.__rowcount = lib4d_sql.fourd_affected_rows(self.fourdconn);
if self.__resulttype == self.lib4d_sql.RESULT_SET:
self.__rowcount = self.lib4d_sql.fourd_num_rows(self.result)
elif self.__resulttype == self.lib4d_sql.UPDATE_COUNT:
self.__rowcount = self.lib4d_sql.fourd_affected_rows(self.fourdconn);
else:
self.__rowcount = -1 # __resulttype is an enum, so this shouldn't happen.

Expand All @@ -317,27 +317,27 @@ def __describe(self):
if self.result == ffi.NULL:
return

columncount = lib4d_sql.fourd_num_columns(self.result)
columncount = self.lib4d_sql.fourd_num_columns(self.result)

description = []
pythonTypes = {lib4d_sql.VK_BOOLEAN: bool,
lib4d_sql.VK_BYTE: str,
lib4d_sql.VK_WORD: str,
lib4d_sql.VK_LONG: int,
lib4d_sql.VK_LONG8: int,
lib4d_sql.VK_REAL: float,
lib4d_sql.VK_FLOAT: float,
lib4d_sql.VK_TIME: time,
lib4d_sql.VK_TIMESTAMP: datetime,
lib4d_sql.VK_DURATION: timedelta,
lib4d_sql.VK_TEXT: str,
lib4d_sql.VK_STRING: str,
lib4d_sql.VK_BLOB: Binary,
lib4d_sql.VK_IMAGE: Binary,}
pythonTypes = {self.lib4d_sql.VK_BOOLEAN: bool,
self.lib4d_sql.VK_BYTE: str,
self.lib4d_sql.VK_WORD: str,
self.lib4d_sql.VK_LONG: int,
self.lib4d_sql.VK_LONG8: int,
self.lib4d_sql.VK_REAL: float,
self.lib4d_sql.VK_FLOAT: float,
self.lib4d_sql.VK_TIME: time,
self.lib4d_sql.VK_TIMESTAMP: datetime,
self.lib4d_sql.VK_DURATION: timedelta,
self.lib4d_sql.VK_TEXT: str,
self.lib4d_sql.VK_STRING: str,
self.lib4d_sql.VK_BLOB: Binary,
self.lib4d_sql.VK_IMAGE: Binary,}

for colidx in range(columncount):
colName = ffi.string(lib4d_sql.fourd_get_column_name(self.result, colidx))
colType = lib4d_sql.fourd_get_column_type(self.result, colidx)
colName = ffi.string(self.lib4d_sql.fourd_get_column_name(self.result, colidx))
colType = self.lib4d_sql.fourd_get_column_type(self.result, colidx)
try:
pytype = pythonTypes[colType]
except KeyError:
Expand All @@ -353,6 +353,7 @@ def executemany(self, query, params):
""""""
for paramlist in params:
self.execute(query, paramlist, describe=False)
self.lib4d_sql.fourd_close_statement(self.result)
self.__prepared = True

#we don't run describe on the individual queries in order to be more efficent.
Expand All @@ -368,44 +369,44 @@ def fetchone(self):
if self.__resulttype is None:
raise DataError("No rows to fetch")

if self.rowcount == 0 or self.__resulttype == lib4d_sql.UPDATE_COUNT:
if self.rowcount == 0 or self.__resulttype == self.lib4d_sql.UPDATE_COUNT:
return None

# get the next row of the result set
#if self.rownumber >= self.result.row_count_sent - 1:
# return None #no more results have been returned

goodrow = lib4d_sql.fourd_next_row(self.result)
goodrow = self.lib4d_sql.fourd_next_row(self.result)
if goodrow == 0:
return None

self.__rownumber = self.result.numRow

numcols = lib4d_sql.fourd_num_columns(self.result);
numcols = self.lib4d_sql.fourd_num_columns(self.result);
inbuff = ffi.new("char*[1024]")
strlen = ffi.new("size_t*")

row=[]
for col in range(numcols):
fieldtype=lib4d_sql.fourd_get_column_type(self.result,col)
if lib4d_sql.fourd_field(self.result,col)==ffi.NULL: #shouldn't happen, really. but handle just in case.
fieldtype=self.lib4d_sql.fourd_get_column_type(self.result,col)
if self.lib4d_sql.fourd_field(self.result,col)==ffi.NULL: #shouldn't happen, really. but handle just in case.
row.append(None)
continue

lib4d_sql.fourd_field_to_string(self.result, col, inbuff, strlen)
self.lib4d_sql.fourd_field_to_string(self.result, col, inbuff, strlen)
output = str(ffi.buffer(inbuff[0], strlen[0])[:])

if fieldtype==lib4d_sql.VK_STRING or fieldtype==lib4d_sql.VK_TEXT:
if fieldtype==self.lib4d_sql.VK_STRING or fieldtype==self.lib4d_sql.VK_TEXT:
row.append(output.decode('UTF-16LE'))
elif fieldtype == lib4d_sql.VK_BOOLEAN:
boolval = lib4d_sql.fourd_field_long(self.result, col)
elif fieldtype == self.lib4d_sql.VK_BOOLEAN:
boolval = self.lib4d_sql.fourd_field_long(self.result, col)
row.append(bool(boolval[0]))
elif fieldtype == lib4d_sql.VK_LONG or fieldtype == lib4d_sql.VK_LONG8:
intval = lib4d_sql.fourd_field_long(self.result, col)
elif fieldtype == self.lib4d_sql.VK_LONG or fieldtype == self.lib4d_sql.VK_LONG8:
intval = self.lib4d_sql.fourd_field_long(self.result, col)
row.append(intval[0])
elif fieldtype == lib4d_sql.VK_REAL or fieldtype == lib4d_sql.VK_FLOAT:
elif fieldtype == self.lib4d_sql.VK_REAL or fieldtype == self.lib4d_sql.VK_FLOAT:
row.append(float(output))
elif fieldtype == lib4d_sql.VK_TIMESTAMP:
elif fieldtype == self.lib4d_sql.VK_TIMESTAMP:
if output == '0000/00/00 00:00:00.000':
dateval = None
else:
Expand All @@ -418,15 +419,15 @@ def fetchone(self):
except:
dateval = None
row.append(dateval)
elif fieldtype == lib4d_sql.VK_DURATION:
elif fieldtype == self.lib4d_sql.VK_DURATION:
#milliseconds from midnight
longval = lib4d_sql.fourd_field_long(self.result, col)
longval = self.lib4d_sql.fourd_field_long(self.result, col)
durationval = timedelta(milliseconds=longval[0])
midnight = datetime(1, 1, 1) #we are going to ignore the date anyway
timeval = midnight + durationval
row.append(timeval.time())
elif fieldtype == lib4d_sql.VK_BLOB or fieldtype == lib4d_sql.VK_IMAGE:
field = lib4d_sql.fourd_field(self.result, col)
elif fieldtype == self.lib4d_sql.VK_BLOB or fieldtype == self.lib4d_sql.VK_IMAGE:
field = self.lib4d_sql.fourd_field(self.result, col)
if field != ffi.NULL:
field = ffi.cast("FOURD_BLOB *", field)
fieldlen = field.length
Expand Down
1 change: 1 addition & 0 deletions p4d/py_fourd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ int fourd_close(FOURD *cnx);
FOURD_STATEMENT * fourd_prepare_statement(FOURD *cnx,const char *query);
int fourd_bind_param(FOURD_STATEMENT *state,unsigned int numParam,FOURD_TYPE type, void *val);
FOURD_RESULT *fourd_exec_statement(FOURD_STATEMENT *state, int res_size);
int fourd_close_statement(FOURD_RESULT *res);

int fourd_next_row(FOURD_RESULT *res);
void * fourd_field(FOURD_RESULT *res,unsigned int numCol);
Expand Down
2 changes: 1 addition & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def finalize_options(self):
setup(
zip_safe=False,
name="p4d",
version="0.3",
version="0.3.2",
install_requires=["cffi", ],
setup_requires=['cffi', ],
packages=find_packages(),
Expand Down

0 comments on commit 4bb392c

Please sign in to comment.