Enable parallel training UT in GitHub CI. (#1075)

* Remove dupliated setting of `allow_growth` in trainer.

* Make parallel training UT independent of its working folder.

* Skip parallel-training tests when there is only 1 GPU card.

* Enable parallel training UT in GitHub CI.

Author: shishaochen

.github/workflows/test_python.yml

@@ -71,4 +71,9 @@ jobs:
         CXX: g++-${{ matrix.gcc }}
         TENSORFLOW_VERSION: ${{ matrix.tf }}
     - run: dp --version
+    - name: Prepare parallel runtime
+      if: ${{ matrix.tf == '' }}
+      run: |
+        sudo apt install libopenmpi-dev openmpi-bin
+        HOROVOD_WITHOUT_GLOO=1 HOROVOD_WITH_TENSORFLOW=1 pip install horovod mpi4py
     - run: pytest --cov=deepmd source/tests && codecov

deepmd/train/trainer.py

@@ -397,7 +397,6 @@ def _init_session(self):
         config = get_tf_session_config()
         device, idx = self.run_opt.my_device.split(":", 1)
         if device == "gpu":
-            config.gpu_options.allow_growth = True
             config.gpu_options.visible_device_list = idx
         self.sess = tf.Session(config=config)
 

source/tests/test_parallel_training.py

@@ -18,9 +18,12 @@ def setUp(self):
     def test_two_workers(self):
         command = 'horovodrun -np 2 dp train -m workers ' + self.input_file
         penv = os.environ.copy()
-        if len(get_gpus() or []) > 1:
+        num_gpus = len(get_gpus() or [])
+        if num_gpus > 1:
             penv['CUDA_VISIBLE_DEVICES'] = '0,1'
-        popen = sp.Popen(command, shell=True, env=penv, stdout=sp.PIPE, stderr=sp.STDOUT)
+        elif num_gpus == 1:
+            raise unittest.SkipTest("At least 2 GPU cards are needed for parallel-training tests.")
+        popen = sp.Popen(command, shell=True, cwd=str(tests_path), env=penv, stdout=sp.PIPE, stderr=sp.STDOUT)
         for line in iter(popen.stdout.readline, b''):
             if hasattr(line, 'decode'):
                 line = line.decode('utf-8')