Make Series.astype(bool) follow the concept of "truthy" and "falsey". (#1431)

Making `Series.astype(bool)` follow the concept of "truthy" and "falsey".

```py
>>> kser = ks.Series(["hi", "hi ", " ", " \t", "", None], name="x")
>>> kser
0      hi
1     hi
2
3      \t
4
5    None
Name: x, dtype: object
>>> kser.astype(bool)
0     True
1     True
2     True
3     True
4    False
5    False
Name: x, dtype: bool
>>> kser.str.strip().astype(bool)
0     True
1     True
2    False
3    False
4    False
5    False
Name: x, dtype: bool
```

Resolves #1430.

Author: ueshin

databricks/koalas/series.py

@@ -32,7 +32,15 @@
 from databricks.koalas.typedef import as_python_type
 from pyspark import sql as spark
 from pyspark.sql import functions as F, Column
-from pyspark.sql.types import BooleanType, StructType, LongType, IntegerType
+from pyspark.sql.types import (
+    BooleanType,
+    DoubleType,
+    FloatType,
+    StringType,
+    StructType,
+    LongType,
+    IntegerType,
+)
 from pyspark.sql.window import Window
 
 from databricks import koalas as ks  # For running doctests and reference resolution in PyCharm.
@@ -918,7 +926,20 @@ def astype(self, dtype) -> "Series":
         spark_type = as_spark_type(dtype)
         if not spark_type:
             raise ValueError("Type {} not understood".format(dtype))
-        return self._with_new_scol(self._scol.cast(spark_type))
+        if isinstance(spark_type, BooleanType):
+            if isinstance(self.spark_type, StringType):
+                scol = F.when(self._scol.isNull(), F.lit(False)).otherwise(F.length(self._scol) > 0)
+            elif isinstance(self.spark_type, (FloatType, DoubleType)):
+                scol = F.when(self._scol.isNull() | F.isnan(self._scol), F.lit(True)).otherwise(
+                    self._scol.cast(spark_type)
+                )
+            else:
+                scol = F.when(self._scol.isNull(), F.lit(False)).otherwise(
+                    self._scol.cast(spark_type)
+                )
+        else:
+            scol = self._scol.cast(spark_type)
+        return self._with_new_scol(scol)
 
     def getField(self, name):
         if not isinstance(self.spark_type, StructType):

databricks/koalas/tests/test_series.py

@@ -1025,6 +1025,26 @@ def test_shift(self):
     def test_astype(self):
         pser = pd.Series([10, 20, 15, 30, 45], name="x")
         kser = ks.Series(pser)
+
+        self.assert_eq(kser.astype(int), pser.astype(int))
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+
+        pser = pd.Series([10, 20, 15, 30, 45, None, np.nan], name="x")
+        kser = ks.Series(pser)
+
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+
+        pser = pd.Series(["hi", "hi ", " ", " \t", "", None], name="x")
+        kser = ks.Series(pser)
+
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+        self.assert_eq(kser.str.strip().astype(bool), pser.str.strip().astype(bool))
+
+        pser = pd.Series([True, False, None], name="x")
+        kser = ks.Series(pser)
+
+        self.assert_eq(kser.astype(bool), pser.astype(bool))
+
         with self.assertRaisesRegex(ValueError, "Type int63 not understood"):
             kser.astype("int63")
 

databricks/koalas/typedef.py

@@ -280,7 +280,7 @@ def clean_fun(*args2):
     series = kser._with_new_scol(scol=col)  # type: 'ks.Series'
     all_name_tokens = name_tokens + sorted(kw_name_tokens)
     name = "{}({})".format(f.__name__, ", ".join(all_name_tokens))
-    series = series.astype(return_type).rename(name)
+    series = series.rename(name)
     return series