Add python interface to bandwidth estimator (#67)

* Add cmdinfer

Signed-off-by: Ze Gan <[email protected]>

* Refactor directory structure

Signed-off-by: Ze Gan <[email protected]>

* Fix azurepipeline

Signed-off-by: Ze Gan <[email protected]>

* Insert python exception traceback to logfile

Signed-off-by: Ze Gan <[email protected]>

* Update Readme

Signed-off-by: Ze Gan <[email protected]>

Author: Pterosaur

Makefile

@@ -5,6 +5,7 @@ output_dir := out/Default
 target_dir := target
 target_lib_dir := $(target_dir)/lib
 target_bin_dir := $(target_dir)/bin
+target_pylib_dir := $(target_dir)/pylib
 
 compile_docker := alphartc-compile
 release_docker := alphartc
@@ -37,7 +38,8 @@ peerconnection_serverless:
 		make docker-$@ \
 		output_dir=$(output_dir) \
 		target_lib_dir=$(target_lib_dir) \
-		target_bin_dir=$(target_bin_dir)
+		target_bin_dir=$(target_bin_dir) \
+		target_pylib_dir=$(target_pylib_dir)
 
 # Docker internal command
 
@@ -51,8 +53,14 @@ docker-app: docker-peerconnection_serverless
 
 docker-peerconnection_serverless:
 	ninja -C $(output_dir) peerconnection_serverless
+
 	mkdir -p $(target_lib_dir)
 	cp modules/third_party/onnxinfer/lib/*.so $(target_lib_dir)
 	cp modules/third_party/onnxinfer/lib/*.so.* $(target_lib_dir)
+
 	mkdir -p $(target_bin_dir)
 	cp $(output_dir)/peerconnection_serverless $(target_bin_dir)
+	cp examples/peerconnection/serverless/peerconnection_serverless_pyinfer $(target_bin_dir)
+
+	mkdir -p $(target_pylib_dir)
+	cp modules/third_party/cmdinfer/*.py $(target_pylib_dir)/

README.md

@@ -39,7 +39,7 @@
 
 AlphaRTC is a fork of Google's WebRTC project using ML-based bandwidth estimation, delivered by the OpenNetLab team. By equipping WebRTC with a more accurate bandwidth estimator, our mission is to eventually increase the quality of transmission.
 
-AlphaRTC replaces Google Congestion Control (GCC) with ONNXInfer, an ML-powered bandwidth estimator, which takes in an ONNX model to make bandwidth estimation more accurate. ONNXInfer is proudly powered by Microsoft's [ONNXRuntime](https://github.com/microsoft/onnxruntime).
+AlphaRTC replaces Google Congestion Control (GCC) with two customized congestion control interfaces, PyInfer and ONNXInfer. The PyInfer provides an opportunity to load external bandwidth estimator written by Python. The external bandwidth estimator could be based on ML framework, like PyTorch or TensorFlow, or a pure Python algorithm without any dependencies. And the ONNXInfer is an ML-powered bandwidth estimator, which takes in an ONNX model to make bandwidth estimation more accurate. ONNXInfer is proudly powered by Microsoft's [ONNXRuntime](https://github.com/microsoft/onnxruntime).
 
 ## Environment
 
@@ -107,7 +107,7 @@ Note: all commands below work for both Linux (sh) and Windows (pwsh), unless oth
     gn gen out/Default
     ```
 
-5. Comile
+5. Compile
     ```shell
     ninja -C out/Default peerconnection_serverless
     ```
@@ -151,9 +151,6 @@ This section describes required fields for the json configuration file.
 
 - **bwe_feedback_duration**: The duration the receiver sends its estimated target rate every time(*in millisecond*)
 
-- **onnx**
-  - **onnx_model_path**: The path of the [onnx](https://www.onnxruntime.ai/) model
-
 - **video_source**
   - **video_disabled**:
     - **enabled**: If set to `true`, the client will not take any video source as input
@@ -188,11 +185,57 @@ This section describes required fields for the json configuration file.
     - **fps**: Frames per second of the output video file
     - **file_path**: The file path of the output video file in YUV format
 
+#### Use PyInfer or ONNXInfer
+
+##### PyInfer
+
+The default bandwidth estimator is PyInfer, You should implement your Python class named `Estimator` with required methods `report_states` and `get_estimated_bandwidth` in Python file `BandwidthEstimator.py ` and put this file in your workspace. And you should use the `peerconnection_serverless_pyinfer` as the start program.
+There is an example of Estimator with fixed estimated bandwidth 1Mbps. Here is an example [BandwidthEstimator.py](examples/peerconnection/serverless/corpus/BandwidthEstimator.py).
+
+```python
+class Estimator(object):
+    def report_states(self, stats: dict):
+        '''
+        stats is a dict with the following items
+        {
+            "send_time_ms": uint,
+            "arrival_time_ms": uint,
+            "payload_type": int,
+            "sequence_number": uint,
+            "ssrc": int,
+            "padding_length": uint,
+            "header_length": uint,
+            "payload_size": uint
+        }
+        '''
+        pass
+
+    def get_estimated_bandwidth(self)->int:
+        return int(1e6) # 1Mbps
+
+```
+
+##### ONNXInfer
+
+If you want to use the ONNXInfer as the bandwidth estimator, you should specify the path of onnx model in the config file. Here is an example configuration [receiver.json](examples/peerconnection/serverless/corpus/receiver.json)
+
+- **onnx**
+  - **onnx_model_path**: The path of the [onnx](https://www.onnxruntime.ai/) model
+
+
 #### Run peerconnection_serverless
 - Dockerized environment
 
     To better demonstrate the usage of peerconnection_serverless, we provide an all-inclusive corpus in `examples/peerconnection/serverless/corpus`. You can use the following commands to execute a tiny example. After these commands terminates, you will get `outvideo.yuv` and `outaudio.wav`.
-    
+
+
+    PyInfer:
+    ```shell
+    sudo docker run -d --rm -v `pwd`/examples/peerconnection/serverless/corpus:/app -w /app --name alphartc alphartc peerconnection_serverless_pyinfer receiver_pyinfer.json
+    sudo docker exec alphartc sender_pyinfer sender.json
+    ```
+
+    ONNXInfer:
     ``` shell
     sudo docker run -d --rm -v `pwd`/examples/peerconnection/serverless/corpus:/app -w /app --name alphartc alphartc peerconnection_serverless receiver.json
     sudo docker exec alphartc peerconnection_serverless sender.json

api/alphacc_config.cc

@@ -68,10 +68,10 @@ bool ParseAlphaCCConfig(const std::string& file_path) {
   RETURN_ON_FAIL(
       GetInt(top, "bwe_feedback_duration", &config->bwe_feedback_duration_ms));
 
-  RETURN_ON_FAIL(GetValue(top, "onnx", &second));
-  RETURN_ON_FAIL(
-      GetString(second, "onnx_model_path", &config->onnx_model_path));
-  second.clear();
+  if (GetValue(top, "onnx", &second)) {
+    GetString(second, "onnx_model_path", &config->onnx_model_path);
+    second.clear();
+  }
 
   bool enabled = false;
   RETURN_ON_FAIL(GetValue(top, "video_source", &second));

azure-pipelines.yml

@@ -33,3 +33,7 @@ steps:
 - script: docker run -d --rm -v `pwd`/examples/peerconnection/serverless/corpus:/app -w /app --name alphartc alphartc peerconnection_serverless receiver.json
             && docker exec alphartc peerconnection_serverless sender.json
   displayName: 'run example'
+
+- script: docker run -d --rm -v `pwd`/examples/peerconnection/serverless/corpus:/app -w /app --name alphartc_pyinfer alphartc peerconnection_serverless_pyinfer receiver_pyinfer.json
+            && docker exec alphartc_pyinfer peerconnection_serverless_pyinfer sender_pyinfer.json
+  displayName: 'run cmdinfer example'

dockers/Dockerfile.release

@@ -1,9 +1,10 @@
 FROM ubuntu:18.04
 
 RUN apt-get update && apt-get install -y \
-    libx11-6 libgomp1
+    libx11-6 libgomp1 python3
 
 COPY lib /usr/lib/
 
 COPY bin /usr/bin/
 
+COPY pylib /usr/lib/python3/dist-packages/

examples/peerconnection/serverless/corpus/BandwidthEstimator.py

@@ -0,0 +1,20 @@
+
+class Estimator(object):
+    def report_states(self, stats: dict):
+        '''
+        stats is a dict with the following items
+        {
+            "send_time_ms": uint,
+            "arrival_time_ms": uint,
+            "payload_type": int,
+            "sequence_number": uint,
+            "ssrc": int,
+            "padding_length": uint,
+            "header_length": uint,
+            "payload_size": uint
+        }
+        '''
+        pass
+
+    def get_estimated_bandwidth(self)->int:
+        return int(1e6) # 1Mbps

examples/peerconnection/serverless/corpus/receiver_pyinfer.json

@@ -0,0 +1,54 @@
+{
+    "serverless_connection": {
+        "autoclose": 20,
+        "sender": {
+            "enabled": false
+        },
+        "receiver": {
+            "enabled": true,
+            "listening_ip": "0.0.0.0",
+            "listening_port": 8000
+        }
+    },
+    "bwe_feedback_duration": 200,
+    "video_source": {
+        "video_disabled": {
+            "enabled": true
+        },
+        "webcam": {
+            "enabled": false
+        },
+        "video_file": {
+            "enabled": false,
+            "height": 240,
+            "width": 320,
+            "fps": 10,
+            "file_path": "testmedia/test.yuv"
+        }
+    },
+    "audio_source": {
+        "microphone": {
+            "enabled": false
+        },
+        "audio_file": {
+            "enabled": true,
+            "file_path": "testmedia/test.wav"
+        }
+    },
+    "save_to_file": {
+        "enabled": true,
+        "audio": {
+            "file_path": "outaudio.wav"
+        },
+        "video": {
+            "width": 320,
+            "height": 240,
+            "fps": 10,
+            "file_path": "outvideo.yuv"
+        }
+    },
+    "logging": {
+        "enabled": true,
+        "log_output_path": "webrtc.log"
+    }
+}

examples/peerconnection/serverless/corpus/sender_pyinfer.json

@@ -0,0 +1,44 @@
+{
+    "serverless_connection": {
+        "autoclose": 20,
+        "sender": {
+            "enabled": true,
+            "dest_ip": "0.0.0.0",
+            "dest_port": 8000
+        },
+        "receiver": {
+            "enabled": false
+        }
+    },
+    "bwe_feedback_duration": 200,
+    "video_source": {
+        "video_disabled": {
+            "enabled": false
+        },
+        "webcam": {
+            "enabled": false
+        },
+        "video_file": {
+            "enabled": true,
+            "height": 240,
+            "width": 320,
+            "fps": 10,
+            "file_path": "testmedia/test.yuv"
+        }
+    },
+    "audio_source": {
+        "microphone": {
+            "enabled": false
+        },
+        "audio_file": {
+            "enabled": true,
+            "file_path": "testmedia/test.wav"
+        }
+    },
+    "save_to_file": {
+        "enabled": false
+    },
+    "logging": {
+        "enabled": false
+    }
+}

examples/peerconnection/serverless/peerconnection_serverless_pyinfer

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+import subprocess
+import traceback
+import json
+
+sys.path.append(os.getcwd())
+
+import cmdinfer
+
+
+def main():
+    app = subprocess.Popen(
+        ["peerconnection_serverless"] + sys.argv[1:],
+        bufsize=1,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+    try:
+        cmdinfer.main(app.stdout, app.stdin)
+        app.wait()
+    except:
+        if len(sys.argv[1:]) == 0:
+            return
+        config_file = sys.argv[1]
+        config_file = json.load(open(config_file, "r"))
+        if "logging" not in config_file:
+            return
+        if "enabled" not in config_file["logging"] or not config_file["logging"]["enabled"]:
+            return
+        app.terminate()
+        app.wait()
+        with open(config_file["logging"]["log_output_path"], "a") as log_file:
+            error_message = traceback.format_exc()
+            error_message = "\n{}".format(error_message)
+            log_file.write(error_message)
+            sys.stderr.write(error_message)
+
+
+if __name__ == "__main__":
+    main()

modules/remote_bitrate_estimator/BUILD.gn

@@ -62,7 +62,8 @@ rtc_library("remote_bitrate_estimator") {
     "../../system_wrappers:metrics",
     "//third_party/abseil-cpp/absl/strings",
     "//third_party/abseil-cpp/absl/types:optional",
-    "//modules/third_party/statcollect:stat_collect"
+    "//modules/third_party/statcollect:stat_collect",
+    "//modules/third_party/cmdinfer:cmdinfer"
   ]
 
   if (is_linux) {