Skip to content
CI/CD Pipeline

Initial release: ML Microstructure Signals #3

Sign in to view logs

Initial release: ML Microstructure Signals

Initial release: ML Microstructure Signals #3

Workflow file for this run

.github/workflows/ci.yml at 64a95e5
name: CI/CD Pipeline
on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main ]
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Run ruff
run: ruff check .
- name: Run black
run: black --check .
- name: Run mypy
run: mypy ml_microstructure/
test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.11', '3.12']
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Run tests
run: pytest --cov=ml_microstructure --cov-report=xml --cov-report=term-missing
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
file: ./coverage.xml
flags: unittests
name: codecov-umbrella
fail_ci_if_error: false
integration-test:
runs-on: ubuntu-latest
needs: [lint, test]
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Run integration tests
run: pytest tests/ -m integration
- name: Test synthetic data generation
run: |
python -c "
from ml_microstructure.data import SyntheticLOBGenerator
generator = SyntheticLOBGenerator(duration_seconds=10)
data = generator.generate_data()
print(f'Generated {len(data)} synthetic snapshots')
"
- name: Test feature extraction
run: |
python -c "
from ml_microstructure.data import SyntheticLOBGenerator, OrderBookProcessor
from ml_microstructure.features import FeaturePipeline
generator = SyntheticLOBGenerator(duration_seconds=10)
snapshots = generator.generate_data()
processor = OrderBookProcessor()
df = processor.process_snapshots(snapshots)
pipeline = FeaturePipeline()
features = pipeline.extract_features(df)
print(f'Extracted {len(features.columns)} features')
"
- name: Test model training
run: |
python -c "
from ml_microstructure.data import SyntheticLOBGenerator, OrderBookProcessor
from ml_microstructure.features import FeaturePipeline
from ml_microstructure.models import ModelFactory, ModelConfig
from ml_microstructure.utils.labeling import LabelGenerator
import pandas as pd
import numpy as np
# Generate data
generator = SyntheticLOBGenerator(duration_seconds=10)
snapshots = generator.generate_data()
processor = OrderBookProcessor()
df = processor.process_snapshots(snapshots)
# Extract features
pipeline = FeaturePipeline()
df_features = pipeline.extract_features(df)
# Generate labels
label_generator = LabelGenerator()
labels = label_generator.generate_labels(df_features)
df_labeled = df_features.copy()
df_labeled['label'] = labels
df_labeled = df_labeled.dropna()
# Prepare features
feature_cols = [col for col in df_labeled.columns if col not in ['timestamp', 'label']]
X = df_labeled[feature_cols]
y = df_labeled['label']
# Train model
config = ModelConfig(model_type='lightgbm')
model = ModelFactory.create_model(config)
model.fit(X, y)
# Make predictions
predictions = model.predict(X)
probabilities = model.predict_proba(X)
print(f'Trained model on {len(X)} samples')
print(f'Predictions shape: {predictions.shape}')
print(f'Probabilities shape: {probabilities.shape}')
"
build:
runs-on: ubuntu-latest
needs: [lint, test, integration-test]
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install build dependencies
run: |
python -m pip install --upgrade pip
pip install build twine
- name: Build package
run: python -m build
- name: Check package
run: twine check dist/*
- name: Upload build artifacts
uses: actions/upload-artifact@v3
with:
name: dist
path: dist/
security:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install safety bandit
- name: Run safety check
run: safety check
- name: Run bandit security check
run: bandit -r ml_microstructure/
documentation:
runs-on: ubuntu-latest
needs: [lint, test]
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Check documentation
run: |
python -c "
import ml_microstructure
print('Package imports successfully')
print(f'Version: {ml_microstructure.__version__}')
"
- name: Test CLI commands
run: |
python -m ml_microstructure.pipeline.train --help || echo "CLI help not available"
python -m ml_microstructure.pipeline.predict --help || echo "CLI help not available"
python -m ml_microstructure.pipeline.evaluate --help || echo "CLI help not available"
python -m ml_microstructure.backtest.run --help || echo "CLI help not available"