Flexibility in the deps (#165)

* update python deps

* add main to RayDP CI branches

* test multiple pyspark versions

* less changes

Author: mjschock

.github/workflows/raydp.yml

@@ -19,16 +19,17 @@ name: RayDP CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [ main, master ]
   pull_request:
-    branches: [ master ]
+    branches: [ main, master ]
 
 jobs:
   build-and-test:
 
     strategy:
       matrix:
         os: [ ubuntu-latest, macos-latest ]
+        pyspark-version: [3.0.0, 3.1.2]
         python-version: [3.6, 3.7, 3.8]
 
     runs-on: ${{ matrix.os }}
@@ -51,7 +52,6 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt-get install -y mpich
-
       - name: Cache pip - Ubuntu
         if: matrix.os == 'ubuntu-latest'
         uses: actions/cache@v2
@@ -87,6 +87,7 @@ jobs:
         env:
           GITHUB_CI: 1
         run: |
+          pip install pyspark==${{ matrix.pyspark-version }}
           ./build.sh
           pip install dist/raydp-*.whl
       - name: Lint

core/pom.xml

@@ -152,7 +152,7 @@
       <plugin>
         <groupId>net.alchim31.maven</groupId>
         <artifactId>scala-maven-plugin</artifactId>
-        <version>3.2.2</version>
+        <version>3.3.3</version>
         <executions>
           <execution>
             <id>scala-compile-first</id>

python/setup.py

@@ -92,11 +92,11 @@ def run(self):
     install_requires = [
         "numpy",
         "typing",
-        "pandas == 1.1.4",
+        "pandas >= 1.1.4",
         "psutil",
         "pyarrow >= 0.10",
-        "ray == 1.4.0",
-        "pyspark >= 3.0.0, < 3.1.0",
+        "ray >= 1.4.0, < 1.5.0",
+        "pyspark >= 3.0.0",
         "netifaces"
     ]