Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# mac junk
.DS_Store


tree.txt

# visual studio code junk
.vscode/
.vscode/settings.json
Expand Down
84 changes: 60 additions & 24 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ Ivan Wang, [Harrison Gao](https://github.com/HTK-G), [Sina Liu](https://github.c
This folder contains the **machine-learning-client** subsystem of our 3-container project:

- **Machine Learning Client** → collects sensor data (webcam), performs gesture recognition with MediaPipe + PyTorch, and later sends results to MongoDB.
- **Web App** (TBD) → visualizes gesture events stored in the database.
- **Web App** → visualizes gesture events stored in the database.
- **MongoDB** → central datastore for gesture metadata.

The ML client runs entirely as a _backend service_ (no user-facing UI).
Expand All @@ -23,31 +23,67 @@ It processes camera input, performs ML inference, and will later communicate wit

# 1. Project Structure

## Project Structure — Machine Learning Client
## Project Structure

```text
machine-learning-client/
├── data/
│ ├── hagrid_keypoints_X.npy # extracted keypoint features (N x 63)
│ ├── hagrid_keypoints_y.npy # integer labels (N,)
│ └── hagrid_classes.json # gesture class list
├── models/
│ ├── gesture_mlp.pt # trained PyTorch MLP model (.pt)
│ └── train_mlp.py # training script for the MLP
├── src/
│ ├── extract_keypoints_from_hagrid.py # offline feature extractor (HaGRID → keypoints)
│ ├── live_mediapipe_mlp.py # live webcam demo (MediaPipe + PyTorch)
│ └── ... # other helper scripts (if any)
├── tests/ # pytest unit tests (to be implemented)
├── Pipfile # pipenv dependencies
├── Pipfile.lock
├── .pylintrc # lint rules (used by both local & CI)
└── README.md # documentation for ML Client
├── docker-compose.yml
├── instructions.md
├── LICENSE
├── machine-learning-client
│   ├── data
│   │   ├── hagrid_keypoints_X.npy
│   │   └── hagrid_keypoints_y.npy
│   ├── Dockerfile
│   ├── models
│   │   ├── gesture_mlp.pt
│   │   └── train_mlp.py
│   ├── Pipfile
│   ├── Pipfile.lock
│   ├── src
│   │   ├── __init__.py
│   │   ├── extract_keypoints_from_hagrid.py
│   │   └── live_mediapipe_mlp.py
│   └── tests
│   ├── __init__.py
│   ├── test_extract_keypoints_from_hagrid.py
│   └── test_live_mediapipe_mlp.py
├── README.md
└── web-app
├── app.py
├── Dockerfile
├── Pipfile
├── Pipfile.lock
├── readme.txt
├── static
│   ├── audios
│   │   ├── among_us.mp3
│   │   ├── android_beep.mp3
│   │   ├── bom.mp3
│   │   ├── error.mp3
│   │   ├── playme.mp3
│   │   ├── rick_roll.mp3
│   │   ├── rizz.mp3
│   │   ├── sponge_bob.mp3
│   │   └── uwu.mp3
│   ├── hagrid_classes.json
│   ├── images
│   │   ├── fist.png
│   │   ├── like.png
│   │   ├── ok.png
│   │   ├── one.png
│   │   ├── palm.png
│   │   ├── stop.png
│   │   ├── thinking.png
│   │   ├── three.png
│   │   └── two_up.png
│   ├── script.js
│   └── style.css
├── templates
│   └── index.html
└── tests
├── __init__.py
├── conftest.py
└── test_app.py
```

# 2. Environment Setup (macOS, M-series)
Expand Down
2 changes: 1 addition & 1 deletion machine-learning-client/Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@ tomli = "*"

[dev-packages]
pytest = "*"
pytest-cov = "*"
coverage = "*"
black = "*"
pylint = "*"
tomli = "*"
dill = "*"
exceptiongroup = "*"
pytest-cov = "*"

[requires]
python_version = "3.10"
196 changes: 102 additions & 94 deletions machine-learning-client/Pipfile.lock

Large diffs are not rendered by default.

46 changes: 41 additions & 5 deletions machine-learning-client/src/live_mediapipe_mlp.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,48 @@ def init_db() -> None:
print("[INFO] Initialized capture_control = False")


def should_capture() -> bool:
"""Read capture state from DB (Flask updates this)."""
doc = controls_collection.find_one({"_id": "capture_control"})
if doc is None:
# def should_capture() -> bool:
# """Read capture state from DB (Flask updates this)."""
# doc = controls_collection.find_one({"_id": "capture_control"})
# if doc is None:
# return False
# return bool(doc.get("enabled", False))

# --- Cache capture state to avoid hitting DB every frame ---
# _last_check_time = 0
# _cached_capture_state = False


# def should_capture(rate_limit=0.5):
# """Only hit DB at most once every rate_limit seconds."""
# global _last_check_time, _cached_capture_state

# now = time.time()
# if now - _last_check_time < rate_limit:
# return _cached_capture_state

# doc = controls_collection.find_one({"_id": "capture_control"}, {"enabled": 1})
# _cached_capture_state = bool(doc.get("enabled", False)) if doc else False
# _last_check_time = now
# return _cached_capture_state


def should_capture(_rate_limit: float = 0.5) -> bool:
"""
Check whether gesture capture should be enabled.

For simplicity and testability, this function avoids client-side caching
or rate limiting. It performs a direct MongoDB lookup every time.
"""
# If the database has not been initialized, safely return False
if controls_collection is None:
return False
return bool(doc.get("enabled", False))

# Fetch capture state from the control document
doc = controls_collection.find_one({"_id": "capture_control"}, {"enabled": 1})

# If doc is None or missing the 'enabled' field, treat it as disabled
return bool(doc.get("enabled", False)) if doc else False


class GestureMLP(torch.nn.Module):
Expand Down
46 changes: 23 additions & 23 deletions machine-learning-client/tests/test_live_mediapipe_mlp.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,29 +280,29 @@ def insert_one(self, event):
class TestShouldCapture(unittest.TestCase):
"""Tests for the should_capture() function."""

def test_should_capture_doc_is_none(self):
"""Test the case where the control document doesn't exist."""
# Mock the collection to return None
lm.controls_collection = mock.Mock()
lm.controls_collection.find_one.return_value = None

# This covers line 55-56 in live_mediapipe_mlp.py
self.assertFalse(lm.should_capture())

def test_should_capture_enabled_false(self):
"""Test the case where the 'enabled' key is False or missing."""
lm.controls_collection = mock.Mock()

# Test if doc exists but 'enabled' key is missing
lm.controls_collection.find_one.return_value = {"_id": "capture_control"}
self.assertFalse(lm.should_capture())

# Test if 'enabled' key is explicitly False
lm.controls_collection.find_one.return_value = {
"_id": "capture_control",
"enabled": False,
}
self.assertFalse(lm.should_capture())
# def test_should_capture_doc_is_none(self):
# """Test the case where the control document doesn't exist."""
# # Mock the collection to return None
# lm.controls_collection = mock.Mock()
# lm.controls_collection.find_one.return_value = None

# # This covers line 55-56 in live_mediapipe_mlp.py
# self.assertFalse(lm.should_capture())

# def test_should_capture_enabled_false(self):
# """Test the case where the 'enabled' key is False or missing."""
# lm.controls_collection = mock.Mock()

# Test if doc exists but 'enabled' key is missing
# lm.controls_collection.find_one.return_value = {"_id": "capture_control"}
# self.assertFalse(lm.should_capture())

# # Test if 'enabled' key is explicitly False
# lm.controls_collection.find_one.return_value = {
# "_id": "capture_control",
# "enabled": False,
# }
# self.assertFalse(lm.should_capture())


class TestInitDbErrors(unittest.TestCase):
Expand Down
16 changes: 11 additions & 5 deletions tree.txt
Original file line number Diff line number Diff line change
@@ -1,26 +1,30 @@
.
├── docker-compose.yml
├── instructions.md
├── LICENSE
├── machine-learning-client
│   ├── data
│   │   ├── hagrid_keypoints_X.npy
│   │   └── hagrid_keypoints_y.npy
│   ├── Dockerfile
│   ├── models
│   │   ├── gesture_mlp.pt
│   │   └── train_mlp.py
│   ├── Pipfile
│   ├── Pipfile.lock
│   ├── src
│   │   ├── __init__.py
│   │   ├── extract_keypoints_from_hagrid.py
│   │   └── live_mediapipe_mlp.py
│   ├── test_mongo.py
│   └── tests
│   ├── test_extract_keypoints_from_hagrid_unittest.py
│   └── test_live_mediapipe_mlp_unittest.py
│   ├── __init__.py
│   ├── test_extract_keypoints_from_hagrid.py
│   └── test_live_mediapipe_mlp.py
├── README.md
├── tree.txt
└── web-app
├── app.py
├── Dockerfile
├── Pipfile
├── Pipfile.lock
├── readme.txt
Expand Down Expand Up @@ -50,7 +54,9 @@
│   └── style.css
├── templates
│   └── index.html
├── test_pymongo.py
└── tests
├── __init__.py
├── conftest.py
└── test_app.py

12 directories, 42 files
12 directories, 48 files
1 change: 1 addition & 0 deletions web-app/Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ pytest = "*"
pytest-flask = "*"
black = "*"
pylint = "*"
colorama = "*"

[dev-packages]
black = "*"
Expand Down
11 changes: 10 additions & 1 deletion web-app/Pipfile.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading