61 plain non nullable types (tensorflow#68)

azurecoder · Ivan Gavryliuk · commit 4fe27585789d · 2017-07-05T07:34:47.000+01:00
* added new row indexer for parquet data frame

* updated all tests and code to use DateTimeOffset

* added logical JSON type

* added new dataset handling of rows through pivoting

* Update PlainValuesReader.cs

* built more single responsibility around ParquetReader type to ensure efficient deallocation of resources using IDisposable

* updated reader to look at nulls

* added branches to set type IList as either nullable or non-nullable and done this against the required attribute on the column header

* moved BigDecimal to own file
diff --git a/src/Parquet/File/Values/BigDecimal.cs b/src/Parquet/File/Values/BigDecimal.cs
@@ -0,0 +1,31 @@
+using System.Numerics;
+
+namespace Parquet.File.Values
+{
+   public struct BigDecimal
+   {
+      public decimal Integer { get; set; }
+      public int Scale { get; set; }
+      public int Precision { get; set; }
+
+      public BigDecimal(BigInteger integer, int scale, int precision) : this()
+      {
+         Integer = (decimal) integer;
+         Scale = scale;
+         Precision = precision;
+         while (Scale > 0)
+         {
+            Integer /= 10;
+            Scale -= 1;
+         }
+         Scale = scale;
+      }
+
+      public static explicit operator decimal(BigDecimal bd)
+      {
+         return bd.Integer;
+      }
+
+      // TODO: Add to byte array for writer
+   }
+}
diff --git a/src/Parquet/File/Values/PlainValuesReader.cs b/src/Parquet/File/Values/PlainValuesReader.cs
@@ -9,6 +9,7 @@
 using TType = Parquet.Thrift.Type;
 using System.Runtime.CompilerServices;
 using System.Numerics;
+using System.Reflection;
 
 namespace Parquet.File.Values
 {
@@ -65,7 +66,6 @@ private static void ReadPlainBoolean(byte[] data, IList destination, long maxVal
          int ibit = 0;
          int ibyte = 0;
          byte b = data[0];
-         var destinationTyped = (List<bool?>)destination;
 
          for(int ires = 0; ires < maxValues; ires++)
          {
@@ -76,70 +76,64 @@ private static void ReadPlainBoolean(byte[] data, IList destination, long maxVal
             }
 
             bool set = ((b >> ibit++) & 1) == 1;
-            destinationTyped.Add(set);
+            destination.Add(set);
          }
       }
 
       [MethodImpl(MethodImplOptions.AggressiveInlining)]
       private static void ReadInt32(byte[] data, SchemaElement schema, IList destination)
       {
-         if(schema.Converted_type == ConvertedType.DATE)
+         if (schema.Converted_type == ConvertedType.DATE)
          {
-            List<DateTimeOffset?> destinationTyped = (List<DateTimeOffset?>)destination;
             for (int i = 0; i < data.Length; i += 4)
             {
                int iv = BitConverter.ToInt32(data, i);
-               destinationTyped.Add(new DateTimeOffset(iv.FromUnixTime(), TimeSpan.Zero));
+               destination.Add(new DateTimeOffset(iv.FromUnixTime(), TimeSpan.Zero));
             }
          }
          else
          {
-            List<int?> destinationTyped = (List<int?>)destination;
             for (int i = 0; i < data.Length; i += 4)
             {
                int iv = BitConverter.ToInt32(data, i);
-               destinationTyped.Add(iv);
+               destination.Add(iv);
             }
          }
       }
 
       [MethodImpl(MethodImplOptions.AggressiveInlining)]
       private static void ReadFloat(byte[] data, SchemaElement schema, IList destination)
       {
-         List<float?> destinationTyped = (List<float?>)destination;
          for (int i = 0; i < data.Length; i += 4)
          {
             float iv = BitConverter.ToSingle(data, i);
-            destinationTyped.Add(iv);
+            destination.Add(iv);
          }
       }
 
       [MethodImpl(MethodImplOptions.AggressiveInlining)]
       private static void ReadLong(byte[] data, SchemaElement schema, IList destination)
       {
-         List<long?> destinationTyped = (List<long?>)destination;
          for (int i = 0; i < data.Length; i += 8)
          {
             long lv = BitConverter.ToInt64(data, i);
-            destinationTyped.Add(lv);
+            destination.Add(lv);
          }
       }
 
       [MethodImpl(MethodImplOptions.AggressiveInlining)]
       private static void ReadDouble(byte[] data, SchemaElement schema, IList destination)
       {
-         List<double?> destinationTyped = (List<double?>)destination;
          for (int i = 0; i < data.Length; i += 8)
          {
             double lv = BitConverter.ToDouble(data, i);
-            destinationTyped.Add(lv);
+            destination.Add(lv);
          }
       }
 
       [MethodImpl(MethodImplOptions.AggressiveInlining)]
       private static void ReadFixedLenByteArray(byte[] data, SchemaElement schema, IList destination)
       {
-         List<decimal?> destinationTyped = (List<decimal?>) destination;
          for (int i = 0; i < data.Length; i += schema.Type_length)
          {
             if (schema.Converted_type != ConvertedType.DECIMAL) continue;
@@ -149,21 +143,15 @@ private static void ReadFixedLenByteArray(byte[] data, SchemaElement schema, ILi
             var bigInt = new BigDecimal(new BigInteger(dataNew.Reverse().ToArray()), schema.Scale, schema.Precision);
 
             decimal dc = (decimal) bigInt;
-            destinationTyped.Add(dc);
+            destination.Add(dc);
          }
       }
 
       [MethodImpl(MethodImplOptions.AggressiveInlining)]
       private static void ReadInt96(byte[] data, SchemaElement schema, IList destination)
       {
-#if !SPARK_TYPES
-         List<BigInteger?> destinationTyped = (List<BigInteger?>)destination;
-#else
-         List<DateTimeOffset?> destinationTyped = (List<DateTimeOffset?>)destination;
-#endif
 
-         //todo: this is a sample how to read int96, not tested this yet
-         // todo: need to work this out because Spark is not encoding per spec - working with the Spark encoding instead
+
 #if !SPARK_TYPES
          //var r96 = new List<BigInteger>(data.Length / 12);
 #else
@@ -188,7 +176,7 @@ private static void ReadInt96(byte[] data, SchemaElement schema, IList destinati
             double millis = (double) nanosToInt64 / 1000000D;
             bi = bi.AddMilliseconds(millis);
 #endif
-            destinationTyped.Add(new DateTimeOffset(bi));
+            destination.Add(new DateTimeOffset(bi));
 
          } 
       }
@@ -203,62 +191,28 @@ private void ReadByteArray(byte[] data, SchemaElement schemaElement, IList desti
             schemaElement.Converted_type == ConvertedType.UTF8 || schemaElement.Converted_type == ConvertedType.JSON ||
             _options.TreatByteArrayAsString)
          {
-            List<string> destinationTyped = (List<string>)destination;
             for (int i = 0; i < data.Length;)
             {
                int length = BitConverter.ToInt32(data, i);
                i += 4;        //fast-forward to data
                string s = UTF8.GetString(data, i, length);
                i += length;   //fast-forward to the next element
-               destinationTyped.Add(s);
+               destination.Add(s);
             }
          }
          else
          {
-            List<byte[]> destinationTyped = (List<byte[]>)destination;
             for (int i = 0; i < data.Length;)
             {
                int length = BitConverter.ToInt32(data, i);
                i += 4;        //fast-forward to data
                byte[] ar = new byte[length];
                Array.Copy(data, i, ar, 0, length);
                i += length;   //fast-forward to the next element
-               destinationTyped.Add(ar);
+               destination.Add(ar);
             }
          }
       }
 
-   }
-
-
-   struct BigDecimal
-   {
-      public decimal Integer { get; set; }
-      public int Scale { get; set; }
-      public int Precision { get; set; }
-
-      public BigDecimal(BigInteger integer, int scale, int precision) : this()
-      {
-         Integer = (decimal) integer;
-         Scale = scale;
-         Precision = precision;
-         while (Scale > 0)
-         {
-            Integer /= 10;
-            Scale -= 1;
-         }
-         Scale = scale;
-      }
-
-      public static explicit operator decimal(BigDecimal bd)
-      {
-         return bd.Integer;
-      }
-
-      // TODO: Add to byte array for writer
-
-
-
-
    }
 }
diff --git a/src/Parquet/ParquetColumn.cs b/src/Parquet/ParquetColumn.cs
@@ -189,28 +189,58 @@ internal static IList CreateValuesList(SchemaElement schema, out Type systemType
          switch(schema.Type)
          {
             case TType.BOOLEAN:
-               systemType = typeof(bool?);
-               return new List<bool?>();
+               if (schema.Repetition_type == FieldRepetitionType.OPTIONAL)
+               {
+                  systemType = typeof(bool?);
+                  return new List<bool?>();
+               }
+               systemType = typeof(bool);
+               return new List<bool>();
             case TType.INT32:
                if(schema.Converted_type == ConvertedType.DATE)
                {
-                  systemType = typeof(DateTimeOffset?);
-                  return new List<DateTimeOffset?>();
+                  if (schema.Repetition_type == FieldRepetitionType.OPTIONAL)
+                  {
+                     systemType = typeof(DateTimeOffset?);
+                     return new List<DateTimeOffset?>();
+                  }
+                  systemType = typeof(DateTimeOffset);
+                  return new List<DateTimeOffset>();
                }
                else
                {
-                  systemType = typeof(int?);
-                  return new List<int?>();
+                  if (schema.Repetition_type == FieldRepetitionType.OPTIONAL)
+                  {
+                     systemType = typeof(int?);
+                     return new List<int?>();
+                  }
+                  systemType = typeof(int);
+                  return new List<int>();
                }
             case TType.FLOAT:
-               systemType = typeof(float?);
-               return new List<float?>();
+               if (schema.Repetition_type == FieldRepetitionType.OPTIONAL)
+               {
+                  systemType = typeof(float?);
+                  return new List<float?>();
+               }
+               systemType = typeof(float);
+               return new List<float>();
             case TType.INT64:
-               systemType = typeof(long?);
-               return new List<long?>();
+               if (schema.Repetition_type == FieldRepetitionType.OPTIONAL)
+               {
+                  systemType = typeof(long?);
+                  return new List<long?>();
+               }
+               systemType = typeof(long);
+               return new List<long>();
             case TType.DOUBLE:
-               systemType = typeof(double?);
-               return new List<double?>();
+               if (schema.Repetition_type == FieldRepetitionType.OPTIONAL)
+               {
+                  systemType = typeof(double?);
+                  return new List<double?>();
+               }
+               systemType = typeof(double);
+               return new List<double>();
             case TType.INT96:
 #if !SPARK_TYPES
                systemType = typeof(DateTimeOffset?);
@@ -227,20 +257,35 @@ internal static IList CreateValuesList(SchemaElement schema, out Type systemType
                }
                else
                {
-                  systemType = typeof(bool?);
-                  return new List<bool?>();
+                  if (schema.Repetition_type == FieldRepetitionType.OPTIONAL)
+                  {
+                     systemType = typeof(bool?);
+                     return new List<bool?>();
+                  }
+                  systemType = typeof(bool);
+                  return new List<bool>();
                }
             case TType.FIXED_LEN_BYTE_ARRAY:
                // TODO: Converted type should work differently shouldn't inline in this way
                if (schema.Converted_type == ConvertedType.DECIMAL)
                {
+                  if (schema.Repetition_type == FieldRepetitionType.OPTIONAL)
+                  {
+                     systemType = typeof(decimal?);
+                     return new List<decimal?>();
+                  }
                   systemType = typeof(decimal);
-                  return new List<decimal?>();
+                  return new List<decimal>();
                }
                else
                {
-                  systemType = typeof(byte?[]);
-                  return new List<byte?[]>();
+                  if (schema.Repetition_type == FieldRepetitionType.OPTIONAL)
+                  {
+                     systemType = typeof(byte?[]);
+                     return new List<byte?[]>();
+                  }
+                  systemType = typeof(byte[]);
+                  return new List<byte[]>();
                }
             default:
                throw new NotImplementedException($"type {schema.Type} not implemented");

Original file line number	Diff line number	Diff line change
`@@ -9,6 +9,7 @@`
`9`	`9`	`using TType = Parquet.Thrift.Type;`
`10`	`10`	`using System.Runtime.CompilerServices;`
`11`	`11`	`using System.Numerics;`
	`12`	`+using System.Reflection;`
`12`	`13`
`13`	`14`	`namespace Parquet.File.Values`
`14`	`15`	`{`
`@@ -65,7 +66,6 @@ private static void ReadPlainBoolean(byte[] data, IList destination, long maxVal`
`65`	`66`	`int ibit = 0;`
`66`	`67`	`int ibyte = 0;`
`67`	`68`	`byte b = data[0];`
`68`		`- var destinationTyped = (List<bool?>)destination;`
`69`	`69`
`70`	`70`	`for(int ires = 0; ires < maxValues; ires++)`
`71`	`71`	`{`
`@@ -76,70 +76,64 @@ private static void ReadPlainBoolean(byte[] data, IList destination, long maxVal`
`76`	`76`	`}`
`77`	`77`
`78`	`78`	`bool set = ((b >> ibit++) & 1) == 1;`
`79`		`- destinationTyped.Add(set);`
	`79`	`+ destination.Add(set);`
`80`	`80`	`}`
`81`	`81`	`}`
`82`	`82`
`83`	`83`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`84`	`84`	`private static void ReadInt32(byte[] data, SchemaElement schema, IList destination)`
`85`	`85`	`{`
`86`		`- if(schema.Converted_type == ConvertedType.DATE)`
	`86`	`+ if (schema.Converted_type == ConvertedType.DATE)`
`87`	`87`	`{`
`88`		`- List<DateTimeOffset?> destinationTyped = (List<DateTimeOffset?>)destination;`
`89`	`88`	`for (int i = 0; i < data.Length; i += 4)`
`90`	`89`	`{`
`91`	`90`	`int iv = BitConverter.ToInt32(data, i);`
`92`		`- destinationTyped.Add(new DateTimeOffset(iv.FromUnixTime(), TimeSpan.Zero));`
	`91`	`+ destination.Add(new DateTimeOffset(iv.FromUnixTime(), TimeSpan.Zero));`
`93`	`92`	`}`
`94`	`93`	`}`
`95`	`94`	`else`
`96`	`95`	`{`
`97`		`- List<int?> destinationTyped = (List<int?>)destination;`
`98`	`96`	`for (int i = 0; i < data.Length; i += 4)`
`99`	`97`	`{`
`100`	`98`	`int iv = BitConverter.ToInt32(data, i);`
`101`		`- destinationTyped.Add(iv);`
	`99`	`+ destination.Add(iv);`
`102`	`100`	`}`
`103`	`101`	`}`
`104`	`102`	`}`
`105`	`103`
`106`	`104`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`107`	`105`	`private static void ReadFloat(byte[] data, SchemaElement schema, IList destination)`
`108`	`106`	`{`
`109`		`- List<float?> destinationTyped = (List<float?>)destination;`
`110`	`107`	`for (int i = 0; i < data.Length; i += 4)`
`111`	`108`	`{`
`112`	`109`	`float iv = BitConverter.ToSingle(data, i);`
`113`		`- destinationTyped.Add(iv);`
	`110`	`+ destination.Add(iv);`
`114`	`111`	`}`
`115`	`112`	`}`
`116`	`113`
`117`	`114`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`118`	`115`	`private static void ReadLong(byte[] data, SchemaElement schema, IList destination)`
`119`	`116`	`{`
`120`		`- List<long?> destinationTyped = (List<long?>)destination;`
`121`	`117`	`for (int i = 0; i < data.Length; i += 8)`
`122`	`118`	`{`
`123`	`119`	`long lv = BitConverter.ToInt64(data, i);`
`124`		`- destinationTyped.Add(lv);`
	`120`	`+ destination.Add(lv);`
`125`	`121`	`}`
`126`	`122`	`}`
`127`	`123`
`128`	`124`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`129`	`125`	`private static void ReadDouble(byte[] data, SchemaElement schema, IList destination)`
`130`	`126`	`{`
`131`		`- List<double?> destinationTyped = (List<double?>)destination;`
`132`	`127`	`for (int i = 0; i < data.Length; i += 8)`
`133`	`128`	`{`
`134`	`129`	`double lv = BitConverter.ToDouble(data, i);`
`135`		`- destinationTyped.Add(lv);`
	`130`	`+ destination.Add(lv);`
`136`	`131`	`}`
`137`	`132`	`}`
`138`	`133`
`139`	`134`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`140`	`135`	`private static void ReadFixedLenByteArray(byte[] data, SchemaElement schema, IList destination)`
`141`	`136`	`{`
`142`		`- List<decimal?> destinationTyped = (List<decimal?>) destination;`
`143`	`137`	`for (int i = 0; i < data.Length; i += schema.Type_length)`
`144`	`138`	`{`
`145`	`139`	`if (schema.Converted_type != ConvertedType.DECIMAL) continue;`
`@@ -149,21 +143,15 @@ private static void ReadFixedLenByteArray(byte[] data, SchemaElement schema, ILi`
`149`	`143`	`var bigInt = new BigDecimal(new BigInteger(dataNew.Reverse().ToArray()), schema.Scale, schema.Precision);`
`150`	`144`
`151`	`145`	`decimal dc = (decimal) bigInt;`
`152`		`- destinationTyped.Add(dc);`
	`146`	`+ destination.Add(dc);`
`153`	`147`	`}`
`154`	`148`	`}`
`155`	`149`
`156`	`150`	`[MethodImpl(MethodImplOptions.AggressiveInlining)]`
`157`	`151`	`private static void ReadInt96(byte[] data, SchemaElement schema, IList destination)`
`158`	`152`	`{`
`159`		`-#if !SPARK_TYPES`
`160`		`- List<BigInteger?> destinationTyped = (List<BigInteger?>)destination;`
`161`		`-#else`
`162`		`- List<DateTimeOffset?> destinationTyped = (List<DateTimeOffset?>)destination;`
`163`		`-#endif`
`164`	`153`
`165`		`- //todo: this is a sample how to read int96, not tested this yet`
`166`		`- // todo: need to work this out because Spark is not encoding per spec - working with the Spark encoding instead`
	`154`	`+`
`167`	`155`	`#if !SPARK_TYPES`
`168`	`156`	`//var r96 = new List<BigInteger>(data.Length / 12);`
`169`	`157`	`#else`
`@@ -188,7 +176,7 @@ private static void ReadInt96(byte[] data, SchemaElement schema, IList destinati`
`188`	`176`	`double millis = (double) nanosToInt64 / 1000000D;`
`189`	`177`	`bi = bi.AddMilliseconds(millis);`
`190`	`178`	`#endif`
`191`		`- destinationTyped.Add(new DateTimeOffset(bi));`
	`179`	`+ destination.Add(new DateTimeOffset(bi));`
`192`	`180`
`193`	`181`	`}`
`194`	`182`	`}`
`@@ -203,62 +191,28 @@ private void ReadByteArray(byte[] data, SchemaElement schemaElement, IList desti`
`203`	`191`	`schemaElement.Converted_type == ConvertedType.UTF8 \|\| schemaElement.Converted_type == ConvertedType.JSON \|\|`
`204`	`192`	`_options.TreatByteArrayAsString)`
`205`	`193`	`{`
`206`		`- List<string> destinationTyped = (List<string>)destination;`
`207`	`194`	`for (int i = 0; i < data.Length;)`
`208`	`195`	`{`
`209`	`196`	`int length = BitConverter.ToInt32(data, i);`
`210`	`197`	`i += 4; //fast-forward to data`
`211`	`198`	`string s = UTF8.GetString(data, i, length);`
`212`	`199`	`i += length; //fast-forward to the next element`
`213`		`- destinationTyped.Add(s);`
	`200`	`+ destination.Add(s);`
`214`	`201`	`}`
`215`	`202`	`}`
`216`	`203`	`else`
`217`	`204`	`{`
`218`		`- List<byte[]> destinationTyped = (List<byte[]>)destination;`
`219`	`205`	`for (int i = 0; i < data.Length;)`
`220`	`206`	`{`
`221`	`207`	`int length = BitConverter.ToInt32(data, i);`
`222`	`208`	`i += 4; //fast-forward to data`
`223`	`209`	`byte[] ar = new byte[length];`
`224`	`210`	`Array.Copy(data, i, ar, 0, length);`
`225`	`211`	`i += length; //fast-forward to the next element`
`226`		`- destinationTyped.Add(ar);`
	`212`	`+ destination.Add(ar);`
`227`	`213`	`}`
`228`	`214`	`}`
`229`	`215`	`}`
`230`	`216`
`231`		`- }`
`232`		`-`
`233`		`-`
`234`		`- struct BigDecimal`
`235`		`- {`
`236`		`- public decimal Integer { get; set; }`
`237`		`- public int Scale { get; set; }`
`238`		`- public int Precision { get; set; }`
`239`		`-`
`240`		`- public BigDecimal(BigInteger integer, int scale, int precision) : this()`
`241`		`- {`
`242`		`- Integer = (decimal) integer;`
`243`		`- Scale = scale;`
`244`		`- Precision = precision;`
`245`		`- while (Scale > 0)`
`246`		`- {`
`247`		`- Integer /= 10;`
`248`		`- Scale -= 1;`
`249`		`- }`
`250`		`- Scale = scale;`
`251`		`- }`
`252`		`-`
`253`		`- public static explicit operator decimal(BigDecimal bd)`
`254`		`- {`
`255`		`- return bd.Integer;`
`256`		`- }`
`257`		`-`
`258`		`- // TODO: Add to byte array for writer`
`259`		`-`
`260`		`-`
`261`		`-`
`262`		`-`
`263`	`217`	`}`
`264`	`218`	`}`