Turning Pyspark optional.

Author: igorborgest

Pipfile

@@ -8,6 +8,7 @@ black = "==19.3b0"
 flake8 = "==3.7.7"
 pytest = "==4.3.1"
 twine = "==1.13.0"
+cfn-lint = ">=0.22.0"
 
 [packages]
 boto3 = "==1.9.164"
@@ -16,7 +17,6 @@ pandas = "==0.24.2"
 pyarrow = "==0.13.0"
 tenacity = "==5.0.4"
 pg8000 = "==1.13.1"
-pyspark = "==2.4.3"
 
 [requires]
 python_version = "3.7"

Pipfile.lock

@@ -8,7 +8,7 @@ AWS Data Wrangler aims to fill a gap between AWS Analytics Services (Glue, Athen
 
 ---
 
-*Contents:* **[Use Cases](#Use-Cases)** | **[Installation](#Installation)** | **[Usage](#Usage)** | **[License](#License)**
+*Contents:* **[Use Cases](#Use-Cases)** | **[Installation](#Installation)** | **[Examples](#Examples)** | **[License](#License)**
 
 ---
 
@@ -30,7 +30,7 @@ AWS Data Wrangler aims to fill a gap between AWS Analytics Services (Glue, Athen
 AWS Data Wrangler runs only Python 3.6 and beyond.
 And runs on AWS Lambda, AWS Glue, EC2, on-premises, local, etc.
 
-## Usage
+## Examples
 
 ### Writing Pandas Dataframe to Data Lake
 

README.md

@@ -1,4 +1,4 @@
 __title__ = "awswrangler"
 __description__ = "Utilities for Pandas and Apache Spark on AWS."
-__version__ = "0.0b13"
+__version__ = "0.0b14"
 __license__ = "Apache License 2.0"

awswrangler/__version__.py

@@ -1,5 +1,6 @@
 import os
 import logging
+import importlib
 
 import boto3
 from botocore.config import Config
@@ -9,7 +10,11 @@
 from awswrangler.pandas import Pandas
 from awswrangler.glue import Glue
 from awswrangler.redshift import Redshift
-from awswrangler.spark import Spark
+
+PYSPARK_INSTALLED = False
+if importlib.util.find_spec("pyspark"):
+    PYSPARK_INSTALLED = True
+    from awswrangler.spark import Spark
 
 
 LOGGER = logging.getLogger(__name__)
@@ -216,7 +221,9 @@ def redshift(self):
 
     @property
     def spark(self):
-        if not self._spark:
+        if not PYSPARK_INSTALLED:
+            self._spark = None
+        elif not self._spark:
             self._spark = Spark(session=self)
         return self._spark
 

awswrangler/session.py

@@ -31,5 +31,4 @@
         "tenacity>=5.0.4",
         "pg8000>=1.13.1",
     ],
-    extras_require={"pyspark": ["pyspark>=2.4.3"]},
 )