acoustic echo cancellation

livekit-plugins/livekit-plugins-aec/.gitignore

@@ -0,0 +1,11 @@
+CMakeLists.txt.user
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+Testing
+Makefile
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+_deps

livekit-plugins/livekit-plugins-aec/CMakeLists.txt

@@ -0,0 +1,51 @@
+cmake_minimum_required(VERSION 3.19)
+set(CMAKE_CONFIGURATION_TYPES Debug Release)
+
+project(livekit-aec)
+
+set(CMAKE_CXX_STANDARD 17)
+
+# who use this c++ extension on vscode instead of clangd?
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+include(FetchContent)
+FetchContent_Declare(
+    abseil
+    GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git
+    GIT_TAG        20250127.0
+)
+
+FetchContent_MakeAvailable(abseil)
+
+file(GLOB_RECURSE SRC
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/*.h"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/*.hpp"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/*.c"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp"
+)
+
+add_definitions(-DWEBRTC_APM_DEBUG_DUMP=0)
+
+if(APPLE)
+    add_definitions(-DWEBRTC_MAC -DWEBRTC_POSIX -DRTC_DISABLE_LOGGING)
+elseif(UNIX AND NOT APPLE)
+    add_definitions(-DWEBRTC_LINUX -DWEBRTC_POSIX -DRTC_DISABLE_LOGGING)
+elseif(WIN32)
+    add_definitions(-DWEBRTC_WIN -DRTC_DISABLE_LOGGING)
+endif()
+
+add_library(livekit-aec STATIC ${SRC})
+
+target_include_directories(livekit-aec PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src" ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(livekit-aec PRIVATE absl::base)
+target_include_directories(livekit-aec BEFORE PUBLIC $<BUILD_INTERFACE:${absl_SOURCE_DIR}>)
+
+# Python bindings
+find_package(PythonInterp REQUIRED)
+find_package(pybind11 REQUIRED)
+
+message(STATUS "Using Python: ${PYTHON_EXECUTABLE}")
+
+pybind11_add_module(lkaec_python ${CMAKE_CURRENT_SOURCE_DIR}/aec_python.cc)
+target_link_libraries(lkaec_python PUBLIC livekit-aec)

livekit-plugins/livekit-plugins-aec/README.md

@@ -0,0 +1,4 @@
+# LiveKit Plugins Browser 
+
+Chromium Embedded Framework (CEF) for LiveKit Agents
+

livekit-plugins/livekit-plugins-aec/aec_python.cc

@@ -0,0 +1,65 @@
+#include "aec_python.h"
+#include "api/audio/echo_canceller3_config.h"
+#include "api/environment/environment.h"
+#include "api/environment/environment_factory.h"
+#include "modules/audio_processing/aec3/echo_canceller3.h"
+#include "modules/audio_processing/audio_buffer.h"
+
+#include <cstdint>
+#include <pybind11/functional.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+namespace py = pybind11;
+
+Aec::Aec(const AecOptions &options) : options_(options) {
+  webrtc::Environment env = webrtc::CreateEnvironment();
+
+  aec3_ = new webrtc::EchoCanceller3(
+      env, webrtc::EchoCanceller3Config(), std::nullopt, options.sample_rate,
+      options.num_channels, options.num_channels);
+
+  cap_buf_ = new webrtc::AudioBuffer(options.sample_rate, options.num_channels,
+                                     options.sample_rate, options.num_channels,
+                                     options.sample_rate, options.num_channels);
+
+  rend_buf_ = new webrtc::AudioBuffer(
+      options.sample_rate, options.num_channels, options.sample_rate,
+      options.num_channels, options.sample_rate, options.num_channels);
+}
+
+void Aec::CancelEcho(py::array_t<int16_t> cap,
+                     const py::array_t<int16_t> rend) {
+  webrtc::StreamConfig stream_cfg(options_.sample_rate, options_.num_channels);
+
+  cap_buf_->CopyFrom(cap.mutable_data(), stream_cfg);
+  rend_buf_->CopyFrom(rend.data(), stream_cfg);
+
+  if (options_.sample_rate > 16000) {
+    cap_buf_->SplitIntoFrequencyBands();
+    rend_buf_->SplitIntoFrequencyBands();
+  }
+
+  aec3_->AnalyzeCapture(cap_buf_);
+  aec3_->AnalyzeRender(rend_buf_);
+  aec3_->ProcessCapture(cap_buf_, false);
+
+  if (options_.sample_rate > 16000) {
+    cap_buf_->MergeFrequencyBands();
+  }
+
+  cap_buf_->CopyTo(stream_cfg, cap.mutable_data());
+}
+
+PYBIND11_MODULE(lkaec_python, m) {
+  m.doc() = "Acoustic Echo Cancellation (AEC) for LiveKit Agents";
+
+  py::class_<AecOptions>(m, "AecOptions")
+      .def(py::init<>())
+      .def_readwrite("sample_rate", &AecOptions::sample_rate)
+      .def_readwrite("num_channels", &AecOptions::num_channels);
+
+  py::class_<Aec>(m, "Aec")
+      .def(py::init<const AecOptions &>())
+      .def("cancel_echo", &Aec::CancelEcho);
+}

livekit-plugins/livekit-plugins-aec/aec_python.h

@@ -0,0 +1,30 @@
+#ifndef LKAEC_AGENTS_PYTHON_HPP
+#define LKAEC_AGENTS_PYTHON_HPP
+
+#include "modules/audio_processing/aec3/echo_canceller3.h"
+#include "modules/audio_processing/audio_buffer.h"
+
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+
+struct AecOptions {
+  int sample_rate;
+  int num_channels;
+};
+
+class Aec {
+public:
+  Aec(const AecOptions &options);
+
+  void CancelEcho(py::array_t<int16_t> cap, const py::array_t<int16_t> rend);
+
+private:
+  AecOptions options_;
+  webrtc::EchoCanceller3 *aec3_;
+  webrtc::AudioBuffer *cap_buf_;
+  webrtc::AudioBuffer *rend_buf_;
+};
+
+#endif // LKAEC_AGENTS_PYTHON_HPP

livekit-plugins/livekit-plugins-aec/copy_aec3.py

@@ -0,0 +1,112 @@
+import re
+import sys
+import shutil
+from pathlib import Path
+
+INCLUDE_PATTERN = re.compile(r'#include\s*[<"](.+?)[">]')
+ALLOWED_FOLDERS = {
+    "modules",
+    "api",
+    "rtc_base",
+    "system_wrappers",
+    "rtc_tools",
+    "common_audio",
+}
+SKIP_SUBSTRINGS = ("unittest", "avx2", "mock")
+
+
+def should_skip_file(file_path: Path) -> bool:
+    name_lower = file_path.name.lower()
+    return any(sub in name_lower for sub in SKIP_SUBSTRINGS)
+
+
+MANDATORY_PATHS = [
+    "api/environment",
+]
+
+
+def copy_file_with_deps(
+    root_input: Path, root_output: Path, rel_path: Path, visited: set
+):
+    if rel_path in visited:
+        return
+    visited.add(rel_path)
+
+    in_file = root_input / rel_path
+    if not in_file.is_file():
+        print(f"Warning: File not found (skipped): {in_file}")
+        return
+
+    if should_skip_file(in_file):
+        print(f"Skipping file (due to skip rules): {in_file}")
+        return
+
+    out_file = root_output / rel_path
+    out_file.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(in_file, out_file)
+
+    with in_file.open("r", encoding="utf-8", errors="ignore") as f:
+        for line in f:
+            match = INCLUDE_PATTERN.search(line)
+            if not match:
+                continue
+
+            include_str = match.group(1).strip()
+            include_path = Path(include_str)
+
+            if include_path.is_absolute():
+                continue
+
+            first_part = include_path.parts[0] if include_path.parts else ""
+
+            if first_part == "third_party":
+                print(f"Warning: Include found from third_party: {include_path}")
+
+            if first_part in ALLOWED_FOLDERS:
+                dep_file = root_input / include_path
+                if not dep_file.is_file():
+                    print(f"Warning: Could not find included file: {dep_file}")
+                else:
+                    copy_file_with_deps(root_input, root_output, include_path, visited)
+
+
+def copy_directory(dir_path: Path, root_input: Path, root_output: Path, visited: set):
+    if not dir_path.is_dir():
+        print(f"Warning: Directory not found or not valid: {dir_path}")
+        return
+
+    for file_path in dir_path.rglob("*"):
+        if not file_path.is_file():
+            continue
+
+        if file_path.suffix.lower() in [".h", ".hpp", ".c", ".cc", ".cpp"]:
+            rel_path = file_path.relative_to(root_input)
+            copy_file_with_deps(root_input, root_output, rel_path, visited)
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(f"Usage: {sys.argv[0]} <input_webrtc_root> <output_folder>")
+        sys.exit(1)
+
+    input_webrtc_root = Path(sys.argv[1]).resolve()
+    output_folder = Path(sys.argv[2]).resolve()
+
+    aec3_path = input_webrtc_root / "modules" / "audio_processing" / "aec3"
+    if not aec3_path.is_dir():
+        print(f"Error: The path {aec3_path} does not exist or is not a directory.")
+        sys.exit(1)
+
+    visited_files = set()
+
+    copy_directory(aec3_path, input_webrtc_root, output_folder, visited_files)
+
+    for mandatory_rel in MANDATORY_PATHS:
+        mandatory_path = input_webrtc_root / mandatory_rel
+        copy_directory(mandatory_path, input_webrtc_root, output_folder, visited_files)
+
+    print(f"Done. Copied {len(visited_files)} files into {output_folder}")
+
+
+if __name__ == "__main__":
+    main()

livekit-plugins/livekit-plugins-aec/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "livekit-plugins-aec",
+  "private": true,
+  "version": "0.0.5"
+}

livekit-plugins/livekit-plugins-aec/pyproject.toml

@@ -0,0 +1,9 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.cibuildwheel.macos]
+repair-wheel-command = "" # getting issues with unresolved files
+
+[tool.cibuildwheel]
+before-build = "pip install pybind11[global]"

livekit-plugins/livekit-plugins-aec/setup.py

@@ -0,0 +1,66 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pathlib
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import setuptools
+from setuptools import Extension
+from setuptools.command.build_ext import build_ext
+
+here = pathlib.Path(__file__).parent.resolve()
+about = {}
+with open(os.path.join(here, "livekit", "plugins", "aec", "version.py"), "r") as f:
+    exec(f.read(), about)
+
+
+setuptools.setup(
+    name="livekit-plugins-aec",
+    version=about["__version__"],
+    description="Chromium Embedded Framework (CEF) for LiveKit Agents",
+    long_description=(here / "README.md").read_text(encoding="utf-8"),
+    long_description_content_type="text/markdown",
+    url="https://github.com/livekit/agents",
+    classifiers=[
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: Apache Software License",
+        "Topic :: Multimedia :: Sound/Audio",
+        "Topic :: Multimedia :: Video",
+        "Topic :: Scientific/Engineering :: Artificial Intelligence",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3 :: Only",
+    ],
+    keywords=["webrtc", "realtime", "audio", "video", "livekit"],
+    license="Apache-2.0",
+    ext_modules=[CMakeExtension("lkcef_python")],
+    cmdclass={"build_ext": CMakeBuild},
+    packages=setuptools.find_namespace_packages(include=["livekit.*"]),
+    python_requires=">=3.9.0",
+    install_requires=["livekit-agents>=0.12.3"],
+    package_data={
+        "livekit.plugins.browser": ["py.typed"],
+        "livekit.plugins.browser.resources": ["**", "lkcef_app.app"],
+    },
+    project_urls={
+        "Documentation": "https://docs.livekit.io",
+        "Website": "https://livekit.io/",
+        "Source": "https://github.com/livekit/agents",
+    },
+)