From b2fe98ecb8bc989f038880658abad8fde5451031 Mon Sep 17 00:00:00 2001 From: Karim Bahgat Date: Thu, 8 Sep 2016 23:38:00 +0200 Subject: [PATCH 1/3] Added read/write support for more datatypes, and writing nullvalues Added read and write support for dbf float fieldtype. Date type now also returns and writes datetime.date objects, and can also write a list of yr,month,day for backward compat. Previously "N" numeric was the only supported number type, sometimes resulting in strange string nrs and subsequent saving errors. Also enabled writing QGIS nullvalues, previously only when reading. --- shapefile.py | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/shapefile.py b/shapefile.py index f63d9c8..7171bc3 100644 --- a/shapefile.py +++ b/shapefile.py @@ -18,6 +18,7 @@ import array import tempfile import itertools +from datetime import date # # Constants for shape types @@ -36,6 +37,8 @@ MULTIPOINTM = 28 MULTIPATCH = 31 +MISSING = [None,''] + PYTHON3 = sys.version_info[0] == 3 if PYTHON3: @@ -492,14 +495,15 @@ def __record(self): # deleted record return None record = [] - for (name, typ, size, deci), value in zip(self.fields, - recordContents): + for (name, typ, size, deci), value in zip(self.fields, recordContents): if name == 'DeletionFlag': continue elif not value.strip(): + # why this? better to delete this and leave it up to the field type? record.append(value) continue - elif typ == "N": + elif typ in ("N","F"): + # numeric or float: number stored as a string, right justified, and padded with blanks to the width of the field. value = value.replace(b('\0'), b('')).strip() value = value.replace(b('*'), b('')) # QGIS NULL is all '*' chars if value == b(''): @@ -517,18 +521,24 @@ def __record(self): #not parseable as int, set to None value = None elif typ == b('D'): + # date: 8 bytes - date stored as a string in the format YYYYMMDD. if value.count(b('0')) == len(value): # QGIS NULL is all '0' chars value = None else: try: y, m, d = int(value[:4]), int(value[4:6]), int(value[6:8]) - value = [y, m, d] + value = date(y, m, d) except: value = value.strip() elif typ == b('L'): - value = (value in b('YyTt') and b('T')) or \ - (value in b('NnFf') and b('F')) or b('?') + # logical: 1 byte - initialized to 0x20 (space) otherwise T or F. + if value == " ": + value = None # space means missing or not yet set + else: + value = (value in b('YyTt') and b('T')) or \ + (value in b('NnFf') and b('F')) or b('?') else: + # anything else is forced to string/unicode value = u(value) value = value.strip() record.append(value) @@ -918,11 +928,30 @@ def __dbfRecords(self): for (fieldName, fieldType, size, dec), value in zip(self.fields, record): fieldType = fieldType.upper() size = int(size) - if fieldType.upper() == "N": - value = str(value).rjust(size) + if fieldType in ("N","F"): + # numeric or float: number stored as a string, right justified, and padded with blanks to the width of the field. + if value in MISSING: + value = str("*"*size) # QGIS NULL + else: + value = str(value).rjust(size) + elif fieldType == "D": + # date: 8 bytes - date stored as a string in the format YYYYMMDD. + if isinstance(value, date): + value = b"%s%s%s"%(value.year,value.month,value.day) + elif isinstance(value, list) and len(value) == 3: + value = b"%s%s%s"%value + elif value in MISSING: + value = b('0') * 8 # QGIS NULL for date type + else: + raise ShapefileException("Date values must be either a datetime.date object, a list, or a missing value of None or ''.") elif fieldType == 'L': - value = str(value)[0].upper() + # logical: 1 byte - initialized to 0x20 (space) otherwise T or F. + if value in MISSING: + value = str(' ') # missing is set to space + else: + value = str(value)[0].upper() else: + # anything else is forced to string value = str(value)[:size].ljust(size) if len(value) != size: raise ShapefileException( From 09afe2787686c30a49e20e4a64771640fb84de18 Mon Sep 17 00:00:00 2001 From: Karim Bahgat Date: Fri, 9 Sep 2016 22:54:10 +0200 Subject: [PATCH 2/3] Fixed date to string saving --- shapefile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shapefile.py b/shapefile.py index 7171bc3..4532f39 100644 --- a/shapefile.py +++ b/shapefile.py @@ -937,9 +937,9 @@ def __dbfRecords(self): elif fieldType == "D": # date: 8 bytes - date stored as a string in the format YYYYMMDD. if isinstance(value, date): - value = b"%s%s%s"%(value.year,value.month,value.day) + value = value.strftime("%Y%m%d") elif isinstance(value, list) and len(value) == 3: - value = b"%s%s%s"%value + value = date(*value).strftime("%Y%m%d") elif value in MISSING: value = b('0') * 8 # QGIS NULL for date type else: From ba61854aa7161fd7d4cff12b0fd08b6ec7581bb7 Mon Sep 17 00:00:00 2001 From: Karim Bahgat Date: Sat, 10 Sep 2016 00:09:43 +0200 Subject: [PATCH 3/3] Fixed old boolean bug, and autoset size when adding date and boolean fields --- shapefile.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/shapefile.py b/shapefile.py index 4532f39..ac2b135 100644 --- a/shapefile.py +++ b/shapefile.py @@ -498,10 +498,6 @@ def __record(self): for (name, typ, size, deci), value in zip(self.fields, recordContents): if name == 'DeletionFlag': continue - elif not value.strip(): - # why this? better to delete this and leave it up to the field type? - record.append(value) - continue elif typ in ("N","F"): # numeric or float: number stored as a string, right justified, and padded with blanks to the width of the field. value = value.replace(b('\0'), b('')).strip() @@ -535,8 +531,12 @@ def __record(self): if value == " ": value = None # space means missing or not yet set else: - value = (value in b('YyTt') and b('T')) or \ - (value in b('NnFf') and b('F')) or b('?') + if value in b('YyTt1'): + value = True + elif value in b('NnFf0'): + value = False + else: + value = b('?') else: # anything else is forced to string/unicode value = u(value) @@ -1009,6 +1009,12 @@ def poly(self, parts=[], shapeType=POLYGON, partTypes=[]): def field(self, name, fieldType="C", size="50", decimal=0): """Adds a dbf field descriptor to the shapefile.""" + if fieldType == "D": + size = "8" + decimal = 0 + elif fieldType == "L": + size = "1" + decimal = 0 self.fields.append((name, fieldType, size, decimal)) def record(self, *recordList, **recordDict):