diff --git a/python/transformations/fmu-integration/README.md b/python/transformations/fmu-integration/README.md new file mode 100644 index 00000000..0af9c331 --- /dev/null +++ b/python/transformations/fmu-integration/README.md @@ -0,0 +1,32 @@ +# FMU Integration + +This sample shows how to execute an FMU inside a Quix Streams application. It consumes messages from an input topic, runs the FMU for each message, and publishes enriched results to an output topic. + + +## How to use this sample + +- Add a valid `.fmu` file to the app. It should be compiled for Linux if you plan to run it on Quix. The repo includes `simulink_example_inports.fmu` in this folder as an example. +- Edit the filename near the top of `main.py`: +```python +fmu_filename = "simulink_example_inports.fmu" +``` + +- When the app starts, it prints the FMU model variables with their `valueReference` and `causality` so you know which variables are inputs and outputs. +- Define the FMU_processing function according to your model inputs and outputs. + +## Environment variables + +The code sample uses the following environment variables: + +- **input**: Name of the input topic to listen to. +- **output**: Name of the output topic to write to. + +## Contribute + +Submit forked projects to the Quix [GitHub](https://github.com/quixio/quix-samples) repo. Any new project that we accept will be attributed to you and you'll receive $200 in Quix credit. + +## Open source + +This project is open source under the Apache 2.0 license and available in our [GitHub](https://github.com/quixio/quix-samples) repo. + +Please star us and mention us on social to show your appreciation. \ No newline at end of file diff --git a/python/transformations/fmu-integration/app.yaml b/python/transformations/fmu-integration/app.yaml new file mode 100644 index 00000000..a5c9218d --- /dev/null +++ b/python/transformations/fmu-integration/app.yaml @@ -0,0 +1,17 @@ +name: FMU Integration +language: python +variables: + - name: input + inputType: InputTopic + multiline: false + description: Name of the input topic to listen to. + defaultValue: 2d-vector + - name: output + inputType: OutputTopic + multiline: false + description: Name of the output topic to write to. + defaultValue: fmu-output +dockerfile: dockerfile +runEntryPoint: main.py +defaultFile: main.py +libraryItemId: starter-transformation diff --git a/python/transformations/fmu-integration/dockerfile b/python/transformations/fmu-integration/dockerfile new file mode 100644 index 00000000..692316bb --- /dev/null +++ b/python/transformations/fmu-integration/dockerfile @@ -0,0 +1,28 @@ +FROM python:3.12.5-slim-bookworm + +# Set environment variables for non-interactive setup and unbuffered output +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + PYTHONPATH="/app" + +# Build argument for setting the main app path +ARG MAINAPPPATH=. + +# Set working directory inside the container +WORKDIR /app + +# Copy requirements to leverage Docker cache +COPY "${MAINAPPPATH}/requirements.txt" "${MAINAPPPATH}/requirements.txt" + +# Install dependencies without caching +RUN pip install --no-cache-dir -r "${MAINAPPPATH}/requirements.txt" + +# Copy entire application into container +COPY . . + +# Set working directory to main app path +WORKDIR "/app/${MAINAPPPATH}" + +# Define the container's startup command +ENTRYPOINT ["python3", "main.py"] \ No newline at end of file diff --git a/python/transformations/fmu-integration/library.json b/python/transformations/fmu-integration/library.json new file mode 100644 index 00000000..b1b816e8 --- /dev/null +++ b/python/transformations/fmu-integration/library.json @@ -0,0 +1,35 @@ +{ + "libraryItemId": "fmu-integration-transformation", + "name": "FMU Integration Transformation", + "language": "Python", + "IsHighlighted": false, + "tags": { + "Complexity": ["Medium"], + "Technology": ["Quix Streams", "FMU"], + "Pipeline Stage": ["Transformation"], + "Type": ["Code samples"], + "Popular Subjects": [] + }, + "shortDescription": "Consume data from a topic, run an FMU model on it and publish the enriched results to an output topic", + "DefaultFile": "main.py", + "EntryPoint": "dockerfile", + "RunEntryPoint": "main.py", + "Variables": [ + { + "Name": "input", + "Type": "EnvironmentVariable", + "InputType": "InputTopic", + "Description": "Name of the input topic to listen to", + "DefaultValue": "2d-vector", + "Required": true + }, + { + "Name": "output", + "Type": "EnvironmentVariable", + "InputType": "OutputTopic", + "Description": "Name of the output topic to write to.", + "DefaultValue": "fmu-output", + "Required": true + } + ] +} diff --git a/python/transformations/fmu-integration/main.py b/python/transformations/fmu-integration/main.py new file mode 100644 index 00000000..81377bff --- /dev/null +++ b/python/transformations/fmu-integration/main.py @@ -0,0 +1,72 @@ +from quixstreams import Application +from fmpy import read_model_description, extract, instantiate_fmu, simulate_fmu +import pandas as pd +import numpy as np +import os + +# Check FMU model +fmu_filename = "simulink_example_inports.fmu" # adjust if in another path + +# 1) read model description (variable names, valueReferences, interface type) +print("FMU MODEL:") +md = read_model_description(fmu_filename) +print('FMI version:', md.fmiVersion) +for v in md.modelVariables: + print(v.name, v.valueReference, v.causality) + +# Define matlab function call +def FMU_processing(row: dict): + x = row["x"] + y = row["y"] + theta = np.pi / 4 # 45 degrees in radians + + # Build structured input + input_data = np.array( + [(0.0, x, y, theta)], + dtype=[('time', np.float64), + ('x', np.float64), + ('y', np.float64), + ('theta', np.float64)] + ) + + result = simulate_fmu( + fmu_filename, + start_time=0.0, + stop_time=0.0, + input=input_data + ) + + result_df = pd.DataFrame.from_records(result) + + # Convert to standard Python float for JSON serialization + row["x_new"] = float(result_df["Out1"][0]) + row["y_new"] = float(result_df["Out2"][0]) + + +def main(): + # Setup necessary objects + app = Application( + consumer_group="FMU-Model-Run", + auto_create_topics=True, + auto_offset_reset="earliest" + ) + input_topic = app.topic(name=os.environ["input"]) + output_topic = app.topic(name=os.environ["output"]) + sdf = app.dataframe(topic=input_topic) + + + # Do StreamingDataFrame operations/transformations here + #sdf.print_table() + sdf = sdf.update(FMU_processing) + sdf.print_table() + + # Finish off by writing to the final result to the output topic + sdf.to_topic(output_topic) + + # With our pipeline defined, now run the Application + app.run() + + +# It is recommended to execute Applications under a conditional main +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/python/transformations/fmu-integration/requirements.txt b/python/transformations/fmu-integration/requirements.txt new file mode 100644 index 00000000..c8a121c7 --- /dev/null +++ b/python/transformations/fmu-integration/requirements.txt @@ -0,0 +1,5 @@ +quixstreams==3.14.1 +python-dotenv +pandas +fmpy +numpy \ No newline at end of file diff --git a/python/transformations/fmu-integration/simulink_example_inports.fmu b/python/transformations/fmu-integration/simulink_example_inports.fmu new file mode 100644 index 00000000..c57c77a1 Binary files /dev/null and b/python/transformations/fmu-integration/simulink_example_inports.fmu differ diff --git a/python/transformations/matlab-wheel/README.md b/python/transformations/matlab-wheel/README.md new file mode 100644 index 00000000..5f67c8d8 --- /dev/null +++ b/python/transformations/matlab-wheel/README.md @@ -0,0 +1,41 @@ +# Matlab Wheel + +This code sample demonstrates how to run MATLAB functions in Quix as Python-compatible .whl packages. + +## How to build the wheel + +### 01 - Your MATLAB function +- Ensure you understand the types and number of inputs and outputs of your function. +- Save your .m function file in the compilation-files folder (like the rot.m example). + +### 02 - Compile for Quix +Let's compile the MATLAB function using the Quix compiler: +- Open MATLAB from the **compilation-files** folder. +- Run the `quix_compiler.m` script, replacing the arguments: + ```matlab + quix_compiler('function_name', 'py') +This will generate a folder named py containing the Python-compatible code, as well as the .whl package that we’ll deploy to Quix. + +<### 03 - Update the .whl in the quix app +Replace the existing .whl file in your Quix app with the new one you just built. +⚠️ If the new filename differs from the previous one, make sure to update the requirements.txt file accordingly. + +### 04 - Update main.py +Edit the `matlab_processing` function in `main.py` to accommodate your specific function's input and output variables. +> +## Environment variables + +The code sample uses the following environment variables: + +- **input**: Name of the input topic to listen to. +- **output**: Name of the output topic to write to. + +## Contribute + +Submit forked projects to the Quix [GitHub](https://github.com/quixio/quix-samples) repo. Any new project that we accept will be attributed to you and you'll receive $200 in Quix credit. + +## Open source + +This project is open source under the Apache 2.0 license and available in our [GitHub](https://github.com/quixio/quix-samples) repo. + +Please star us and mention us on social to show your appreciation. \ No newline at end of file diff --git a/python/transformations/matlab-wheel/app.yaml b/python/transformations/matlab-wheel/app.yaml new file mode 100644 index 00000000..605b839b --- /dev/null +++ b/python/transformations/matlab-wheel/app.yaml @@ -0,0 +1,16 @@ +name: Matlab Wheel +language: python +variables: + - name: input + inputType: InputTopic + multiline: false + description: Name of the input topic to listen to. + defaultValue: 2d-vector + - name: output + inputType: OutputTopic + multiline: false + description: Name of the output topic to write to. + defaultValue: matlab-output +dockerfile: dockerfile +runEntryPoint: main.py +defaultFile: main.py diff --git a/python/transformations/matlab-wheel/compilation-files/build_wheel.sh b/python/transformations/matlab-wheel/compilation-files/build_wheel.sh new file mode 100644 index 00000000..17f7b07d --- /dev/null +++ b/python/transformations/matlab-wheel/compilation-files/build_wheel.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Script to build wheel package from the out folder + +set -e + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="$SCRIPT_DIR/out" + +if [ ! -d "$OUT_DIR" ]; then + echo "Error: 'out' directory not found" + exit 1 +fi + +# Find Python executable +PYTHON_EXEC=$(command -v python3 || command -v python) + +if [ -z "$PYTHON_EXEC" ]; then + echo "Error: Python is not installed or not in PATH." + exit 1 +fi + +echo "Using Python at: $PYTHON_EXEC" +"$PYTHON_EXEC" --version + +echo "Installing build tools..." +"$PYTHON_EXEC" -m pip install --upgrade build wheel + +echo "Building wheel..." +cd "$OUT_DIR" +"$PYTHON_EXEC" -m build --wheel + +echo "Wheel built successfully in $OUT_DIR/dist" + +# Copy the generated wheel file(s) to the root script directory +DIST_DIR="$OUT_DIR/dist" + +if [ -d "$DIST_DIR" ]; then + WHEEL_FILES=("$DIST_DIR"/*.whl) + if [ -e "${WHEEL_FILES[0]}" ]; then + echo "Copying wheel file(s) to $SCRIPT_DIR" + cp "$DIST_DIR"/*.whl "$SCRIPT_DIR"/ + echo "Wheel file(s) copied successfully." + else + echo "No .whl files found in $DIST_DIR" + fi +else + echo "Directory $DIST_DIR does not exist." +fi \ No newline at end of file diff --git a/python/transformations/matlab-wheel/compilation-files/quix_compiler.m b/python/transformations/matlab-wheel/compilation-files/quix_compiler.m new file mode 100644 index 00000000..528d5b8f --- /dev/null +++ b/python/transformations/matlab-wheel/compilation-files/quix_compiler.m @@ -0,0 +1,76 @@ +function quix_compiler(function_name, destination_folder, do_zip) + % quix_compiler Compiles a MATLAB function to Python, builds a wheel, and optionally zips the output. + % Inputs: + % function_name - Name of the MATLAB function file (e.g., 'myfunc') + % destination_folder - Destination folder for the compiled output (e.g., 'build_output') + % do_zip - (Optional) Boolean flag: zip the output folder? Default = true + + if nargin < 3 + do_zip = true; + end + + % Define subfolder for mcc output + out_folder = fullfile(destination_folder, 'out'); + + % Step 0: Create folders if they don’t exist + if ~exist(out_folder, 'dir') + mkdir(out_folder); + end + + % Step 1: Compile using mcc into destination_folder/out + try + fprintf('Compiling %s.m into %s...\n', function_name, out_folder); + mcc('-W', ['python:quixmatlab,' function_name], ... + '-d', out_folder, ... + [function_name, '.m']); + catch ME + error('Compilation failed: %s', ME.message); + end + + % Step 2: Copy build_wheel.sh to destination_folder + script_name = 'build_wheel.sh'; + if exist(script_name, 'file') == 2 + try + copyfile(script_name, destination_folder); + fprintf('Copied %s to %s.\n', script_name, destination_folder); + catch ME + error('Failed to copy script: %s', ME.message); + end + else + warning('%s not found in current directory.\n', script_name); + end + + % === Step 3: Check if pip is installed, if not, install it === + [pip_status, ~] = system('python3 -m pip --version'); + if pip_status ~= 0 + fprintf('pip not found. Installing with get-pip.py...\n'); + system('curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py'); + system('python3 get-pip.py'); + delete('get-pip.py'); % Clean up + else + fprintf('pip is already installed.\n'); + end + + % === Step 4: Run build_wheel.sh in destination_folder === + fprintf('Running build_wheel.sh in %s...\n', destination_folder); + build_cmd = sprintf('cd "%s" && bash build_wheel.sh', destination_folder); + build_status = system(build_cmd); + if build_status ~= 0 + error('build_wheel.sh failed to run correctly.'); + end + + % Step 5: Optionally zip the destination_folder + if do_zip + zip_name = [destination_folder, '.zip']; + try + zip(zip_name, destination_folder); + fprintf('Created zip file: %s\n', zip_name); + catch ME + error('Failed to create zip: %s', ME.message); + end + else + fprintf('Skipping zipping step as requested.\n'); + end + + fprintf('All tasks completed successfully.\n'); +end \ No newline at end of file diff --git a/python/transformations/matlab-wheel/compilation-files/rot.m b/python/transformations/matlab-wheel/compilation-files/rot.m new file mode 100644 index 00000000..b87593b9 --- /dev/null +++ b/python/transformations/matlab-wheel/compilation-files/rot.m @@ -0,0 +1,4 @@ +function M = rot(v, theta) + R = [cos(theta) -sin(theta); sin(theta) cos(theta)]; + M = R * v; +end diff --git a/python/transformations/matlab-wheel/dockerfile b/python/transformations/matlab-wheel/dockerfile new file mode 100644 index 00000000..f363bbbc --- /dev/null +++ b/python/transformations/matlab-wheel/dockerfile @@ -0,0 +1,45 @@ +FROM containers.mathworks.com/matlab-runtime:r2024b + +USER root +# Add MATLAB Runtime +ENV LD_LIBRARY_PATH=/opt/matlabruntime/R2024b/runtime/glnxa64/:$LD_LIBRARY_PATH +ENV AGREE_TO_MATLAB_RUNTIME_LICENSE=yes + +# Install python3.12, venv, and pip +RUN apt-get update && apt-get install -y python3.12 python3.12-venv python3-pip + +# Create virtual environment +ENV VENV_PATH=/opt/venv +RUN python3.12 -m venv $VENV_PATH + +# Add venv to PATH +ENV PATH="$VENV_PATH/bin:$PATH" + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + PYTHONPATH="/app" + +ARG MAINAPPPATH=. + + +# Set working directory +WORKDIR /app + +# === COPY THE REST OF THE APP === +COPY . . + +# ⚠️ Do NOT include matlabengine in requirements.txt +#RUN grep -v "matlabengine" /app/requirements.txt > /app/clean-requirements.txt + +# Install Python dependencies +RUN pip install --no-cache-dir -r /app/requirements.txt + + + +# Set final working directory +WORKDIR "/app/${MAINAPPPATH}" + +# Launch app with virtualenv Python +ENTRYPOINT ["python", "/app/main.py"] \ No newline at end of file diff --git a/python/transformations/matlab-wheel/library.json b/python/transformations/matlab-wheel/library.json new file mode 100644 index 00000000..7b6680a5 --- /dev/null +++ b/python/transformations/matlab-wheel/library.json @@ -0,0 +1,44 @@ +{ + "libraryItemId": "matlab-wheel-transformation", + "name": "MATLAB Wheel Transformation", + "language": "Python", + "IsHighlighted": false, + "tags": { + "Complexity": ["Advanced"], + "Technology": ["QuixStreams", "Matlab"], + "Type": ["Code samples"], + "Pipeline Stage": ["Transformation"], + "Popular Subjects": ["MathWorks", "Scientific Computing"] + }, + "shortDescription": "Transform Quix streams using MATLAB functions as .whl packages", + "longDescription": "Run MATLAB and Simulink models on Quix using the MATLAB runtime for Python", + "DefaultFile": "main.py", + "EntryPoint": "dockerfile", + "RunEntryPoint": "main.py", + "Variables": [ + { + "Name": "input", + "Type": "EnvironmentVariable", + "InputType": "InputTopic", + "Description": "Input topic.", + "DefaultValue": "input-topic", + "Required": true + }, + { + "Name": "output", + "Type": "EnvironmentVariable", + "InputType": "OutputTopic", + "Description": "Output topic.", + "DefaultValue": "output-topic", + "Required": true + } + ], + "DeploySettings": { + "DeploymentType": "Service", + "CpuMillicores": 2000, + "MemoryInMb": 4000, + "Replicas": 1, + "PublicAccess": false, + "ValidateConnection": false + } +} \ No newline at end of file diff --git a/python/transformations/matlab-wheel/main.py b/python/transformations/matlab-wheel/main.py new file mode 100644 index 00000000..83761436 --- /dev/null +++ b/python/transformations/matlab-wheel/main.py @@ -0,0 +1,52 @@ +from quixstreams import Application +import numpy as np +import os +import quixmatlab + +# Initiate quixmatlab +quixmatlab_client = quixmatlab.initialize() +print("Exported MATLAB functions:", dir(quixmatlab_client)) + +# Define matlab function call +def matlab_processing(row: dict): + # Prepare function inputs + x = row["x"] + y = row["y"] + v = np.array([[x],[y]]) + theta = np.pi/4 # 45 degrees in radians + + # Call function here + output = quixmatlab_client.rot(v, theta) + + # Incorporating result to row + row["x_new"] = output[0][0] + row["y_new"] = output[1][0] + + +def main(): + # Setup necessary objects + app = Application( + consumer_group="CompilerSDKMatlab2024b_wheel_general", + auto_create_topics=True, + auto_offset_reset="earliest" + ) + input_topic = app.topic(name=os.environ["input"]) + output_topic = app.topic(name=os.environ["output"]) + sdf = app.dataframe(topic=input_topic) + + + # Do StreamingDataFrame operations/transformations here + sdf.print_table() + sdf = sdf.update(matlab_processing) + sdf.print_table() + + # Finish off by writing to the final result to the output topic + sdf.to_topic(output_topic) + + # With our pipeline defined, now run the Application + app.run() + + +# It is recommended to execute Applications under a conditional main +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/python/transformations/matlab-wheel/quixmatlab_r2024b-24.2-py3-none-any.whl b/python/transformations/matlab-wheel/quixmatlab_r2024b-24.2-py3-none-any.whl new file mode 100644 index 00000000..036f6e92 Binary files /dev/null and b/python/transformations/matlab-wheel/quixmatlab_r2024b-24.2-py3-none-any.whl differ diff --git a/python/transformations/matlab-wheel/requirements.txt b/python/transformations/matlab-wheel/requirements.txt new file mode 100644 index 00000000..e828cf97 --- /dev/null +++ b/python/transformations/matlab-wheel/requirements.txt @@ -0,0 +1,6 @@ +quixstreams==3.17 +numpy +python-dotenv +setuptools +wheel +./quixmatlab_r2024b-24.2-py3-none-any.whl \ No newline at end of file diff --git a/python/transformations/simulink-wheel/README.md b/python/transformations/simulink-wheel/README.md new file mode 100644 index 00000000..0eb5404f --- /dev/null +++ b/python/transformations/simulink-wheel/README.md @@ -0,0 +1,56 @@ +# Simulink Wheel + +This code sample demonstrates how to run Simulink models in Quix as Python-compatible .whl packages. + +## How to build the wheel + +### 01 - Modify Your Simulink Model +- Create one **Inport** block per input signal. The port numbers define the input order. +- Add **Outport** blocks for the signals you want to output. +- Configure **Data Import/Export** settings: + - **Load from workspace**: all options **unchecked** + - **Save to workspace or file**: + - ✅ Output → saved as `yout` + - ✅ Data stores → saved as `dsmout` + - ✅ Single simulation output → saved as `out` +- Save the model after applying these settings. + +### 02 - Wrap the Simulink Model in a MATLAB function +- Save your Simulink model file in the `aux-files` folder (like the rot.m example). +- Open MATLAB from the `aux-files` folder. +- Open `simulink_wrapper.m` and set the `mdl` variable (first line) to your Simulink model's name. +- Create an `inputMatrix`, e.g. `[0, x1, x2, ..., xn]`, where `x1` is the value for Inport 1, and so on, as explained in step 01. + Example: `inputMatrix = [0, 1, 1, pi/2]` +- Run `simulink_wrapper(inputMatrix)` to compile and test the model. Make sure the output order is as expected. + +### 03 - Compile for Quix +Now that the Simulink model is wrapped inside a MATLAB function, you can compile it using the Quix compiler. +- Run the `quix_compiler.m` script, replacing the arguments: + ```matlab + quix_compiler('simulink_wrapper', 'py') +This will generate a folder named py containing the Python-compatible code, as well as the .whl package that we’ll deploy to Quix. + +### 04 - Update the .whl in the quix app +Replace the existing .whl file in your Quix app with the new one you just built. +⚠️ If the new filename differs from the previous one, make sure to update the requirements.txt file accordingly. + +### 05 - Update main.py +Edit the `matlab_processing` function in `main.py` to accommodate your specific function's input and output variables. + + +## Environment variables + +The code sample uses the following environment variables: + +- **input**: Name of the input topic to listen to. +- **output**: Name of the output topic to write to. + +## Contribute + +Submit forked projects to the Quix [GitHub](https://github.com/quixio/quix-samples) repo. Any new project that we accept will be attributed to you and you'll receive $200 in Quix credit. + +## Open source + +This project is open source under the Apache 2.0 license and available in our [GitHub](https://github.com/quixio/quix-samples) repo. + +Please star us and mention us on social to show your appreciation. \ No newline at end of file diff --git a/python/transformations/simulink-wheel/app.yaml b/python/transformations/simulink-wheel/app.yaml new file mode 100644 index 00000000..1857d2b6 --- /dev/null +++ b/python/transformations/simulink-wheel/app.yaml @@ -0,0 +1,16 @@ +name: Simulink Wheel +language: python +variables: + - name: input + inputType: InputTopic + multiline: false + description: Name of the input topic to listen to. + defaultValue: 2d-vector + - name: output + inputType: OutputTopic + multiline: false + description: Name of the output topic to write to. + defaultValue: simulink-output +dockerfile: dockerfile +runEntryPoint: main.py +defaultFile: main.py diff --git a/python/transformations/simulink-wheel/aux-files/build_wheel.sh b/python/transformations/simulink-wheel/aux-files/build_wheel.sh new file mode 100644 index 00000000..17f7b07d --- /dev/null +++ b/python/transformations/simulink-wheel/aux-files/build_wheel.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Script to build wheel package from the out folder + +set -e + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="$SCRIPT_DIR/out" + +if [ ! -d "$OUT_DIR" ]; then + echo "Error: 'out' directory not found" + exit 1 +fi + +# Find Python executable +PYTHON_EXEC=$(command -v python3 || command -v python) + +if [ -z "$PYTHON_EXEC" ]; then + echo "Error: Python is not installed or not in PATH." + exit 1 +fi + +echo "Using Python at: $PYTHON_EXEC" +"$PYTHON_EXEC" --version + +echo "Installing build tools..." +"$PYTHON_EXEC" -m pip install --upgrade build wheel + +echo "Building wheel..." +cd "$OUT_DIR" +"$PYTHON_EXEC" -m build --wheel + +echo "Wheel built successfully in $OUT_DIR/dist" + +# Copy the generated wheel file(s) to the root script directory +DIST_DIR="$OUT_DIR/dist" + +if [ -d "$DIST_DIR" ]; then + WHEEL_FILES=("$DIST_DIR"/*.whl) + if [ -e "${WHEEL_FILES[0]}" ]; then + echo "Copying wheel file(s) to $SCRIPT_DIR" + cp "$DIST_DIR"/*.whl "$SCRIPT_DIR"/ + echo "Wheel file(s) copied successfully." + else + echo "No .whl files found in $DIST_DIR" + fi +else + echo "Directory $DIST_DIR does not exist." +fi \ No newline at end of file diff --git a/python/transformations/simulink-wheel/aux-files/quix_compiler.m b/python/transformations/simulink-wheel/aux-files/quix_compiler.m new file mode 100644 index 00000000..528d5b8f --- /dev/null +++ b/python/transformations/simulink-wheel/aux-files/quix_compiler.m @@ -0,0 +1,76 @@ +function quix_compiler(function_name, destination_folder, do_zip) + % quix_compiler Compiles a MATLAB function to Python, builds a wheel, and optionally zips the output. + % Inputs: + % function_name - Name of the MATLAB function file (e.g., 'myfunc') + % destination_folder - Destination folder for the compiled output (e.g., 'build_output') + % do_zip - (Optional) Boolean flag: zip the output folder? Default = true + + if nargin < 3 + do_zip = true; + end + + % Define subfolder for mcc output + out_folder = fullfile(destination_folder, 'out'); + + % Step 0: Create folders if they don’t exist + if ~exist(out_folder, 'dir') + mkdir(out_folder); + end + + % Step 1: Compile using mcc into destination_folder/out + try + fprintf('Compiling %s.m into %s...\n', function_name, out_folder); + mcc('-W', ['python:quixmatlab,' function_name], ... + '-d', out_folder, ... + [function_name, '.m']); + catch ME + error('Compilation failed: %s', ME.message); + end + + % Step 2: Copy build_wheel.sh to destination_folder + script_name = 'build_wheel.sh'; + if exist(script_name, 'file') == 2 + try + copyfile(script_name, destination_folder); + fprintf('Copied %s to %s.\n', script_name, destination_folder); + catch ME + error('Failed to copy script: %s', ME.message); + end + else + warning('%s not found in current directory.\n', script_name); + end + + % === Step 3: Check if pip is installed, if not, install it === + [pip_status, ~] = system('python3 -m pip --version'); + if pip_status ~= 0 + fprintf('pip not found. Installing with get-pip.py...\n'); + system('curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py'); + system('python3 get-pip.py'); + delete('get-pip.py'); % Clean up + else + fprintf('pip is already installed.\n'); + end + + % === Step 4: Run build_wheel.sh in destination_folder === + fprintf('Running build_wheel.sh in %s...\n', destination_folder); + build_cmd = sprintf('cd "%s" && bash build_wheel.sh', destination_folder); + build_status = system(build_cmd); + if build_status ~= 0 + error('build_wheel.sh failed to run correctly.'); + end + + % Step 5: Optionally zip the destination_folder + if do_zip + zip_name = [destination_folder, '.zip']; + try + zip(zip_name, destination_folder); + fprintf('Created zip file: %s\n', zip_name); + catch ME + error('Failed to create zip: %s', ME.message); + end + else + fprintf('Skipping zipping step as requested.\n'); + end + + fprintf('All tasks completed successfully.\n'); +end \ No newline at end of file diff --git a/python/transformations/simulink-wheel/aux-files/simulink_model_example.slx b/python/transformations/simulink-wheel/aux-files/simulink_model_example.slx new file mode 100644 index 00000000..a099839e Binary files /dev/null and b/python/transformations/simulink-wheel/aux-files/simulink_model_example.slx differ diff --git a/python/transformations/simulink-wheel/aux-files/simulink_wrapper.m b/python/transformations/simulink-wheel/aux-files/simulink_wrapper.m new file mode 100644 index 00000000..77c019b8 --- /dev/null +++ b/python/transformations/simulink-wheel/aux-files/simulink_wrapper.m @@ -0,0 +1,94 @@ +function outputMatrix = simulink_wrapper(inputMatrix) + mdl = "simulink_model_example"; + + %--------------------------------------------------------------- + % 1. Extract time and input signals from input matrix + % Assumes inputMatrix = [time, signal1, signal2, ..., signalN] + %--------------------------------------------------------------- + t = inputMatrix(:,1); % Time vector + u = inputMatrix(:,2:end); % Signal values + n_signals = size(u,2); % Number of input signals + + %--------------------------------------------------------------- + % 2. Build Dataset object + % Timeseries order must match Inport order in the Simulink model + %--------------------------------------------------------------- + inports = Simulink.SimulationData.Dataset; + for i = 1:n_signals + signal = timeseries(u(:,i), t); + inports = inports.addElement(signal); % No name needed + end + + %------------------------------------------------------------------ + % 3. Configure SimulationInput object (efficient for repeated calls) + %------------------------------------------------------------------ + % Use a persistent SimulationInput object to avoid recompilation + persistent s0_loaded + if isempty(s0_loaded) + fprintf("Compiling model...\n") + s0 = Simulink.SimulationInput(mdl); + + % Configure for deployment (Rapid Accelerator + safe options) + % This prepares the model for repeated high-performance execution + s0 = simulink.compiler.configureForDeployment(s0); + + s0_loaded = s0; + fprintf("Compiled\n") + end + + % Clone and update external inputs for this specific run + s = s0_loaded.setExternalInput(inports); + + % Set simulation stop time based on last time sample + s = s.setModelParameter("StopTime", num2str(t(end))); + + %------------------------------------------------------------------ + % 4. Run the simulation + %------------------------------------------------------------------ + out = sim(s); + % Only print sim output once + persistent sim_0 + if isempty(sim_0) + sim_0 = out; + fprintf("Sim ouput:\n") + disp(sim_0) + fprintf("Sim yout:\n") + disp(sim_0.yout) + end + yout = out.yout; + + %--------------------------------------------------------------- + % 5. Extract full output signal values into a matrix [n_times x total_output_width] + %--------------------------------------------------------------- + if isa(yout, 'Simulink.SimulationData.Dataset') + n_outputs = yout.numElements; + outputMatrix = []; + for i = 1:n_outputs + data = yout{i}.Values.Data; + outputMatrix = [outputMatrix, data]; + end + + elseif isnumeric(yout) + % yout is already [n_times x n_outputs] + outputMatrix = yout; + + elseif isstruct(yout) + % Structure with time format + n_outputs = numel(yout.signals); + outputMatrix = []; + for i = 1:n_outputs + data = yout.signals(i).values; + outputMatrix = [outputMatrix, data]; + end + else + error('Unexpected yout type: %s', class(yout)); + end + + persistent outputMatrix_0 + if isempty(outputMatrix_0) + outputMatrix_0 = outputMatrix; + fprintf("First outputMatrix:\n") + disp(outputMatrix_0) + end + +end diff --git a/python/transformations/simulink-wheel/dockerfile b/python/transformations/simulink-wheel/dockerfile new file mode 100644 index 00000000..f363bbbc --- /dev/null +++ b/python/transformations/simulink-wheel/dockerfile @@ -0,0 +1,45 @@ +FROM containers.mathworks.com/matlab-runtime:r2024b + +USER root +# Add MATLAB Runtime +ENV LD_LIBRARY_PATH=/opt/matlabruntime/R2024b/runtime/glnxa64/:$LD_LIBRARY_PATH +ENV AGREE_TO_MATLAB_RUNTIME_LICENSE=yes + +# Install python3.12, venv, and pip +RUN apt-get update && apt-get install -y python3.12 python3.12-venv python3-pip + +# Create virtual environment +ENV VENV_PATH=/opt/venv +RUN python3.12 -m venv $VENV_PATH + +# Add venv to PATH +ENV PATH="$VENV_PATH/bin:$PATH" + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + PYTHONIOENCODING=UTF-8 \ + PYTHONPATH="/app" + +ARG MAINAPPPATH=. + + +# Set working directory +WORKDIR /app + +# === COPY THE REST OF THE APP === +COPY . . + +# ⚠️ Do NOT include matlabengine in requirements.txt +#RUN grep -v "matlabengine" /app/requirements.txt > /app/clean-requirements.txt + +# Install Python dependencies +RUN pip install --no-cache-dir -r /app/requirements.txt + + + +# Set final working directory +WORKDIR "/app/${MAINAPPPATH}" + +# Launch app with virtualenv Python +ENTRYPOINT ["python", "/app/main.py"] \ No newline at end of file diff --git a/python/transformations/simulink-wheel/library.json b/python/transformations/simulink-wheel/library.json new file mode 100644 index 00000000..d5162bb4 --- /dev/null +++ b/python/transformations/simulink-wheel/library.json @@ -0,0 +1,44 @@ +{ + "libraryItemId": "simulink-wheel-transformation", + "name": "Simulink Wheel Transformation", + "language": "Python", + "IsHighlighted": false, + "tags": { + "Complexity": ["Advanced"], + "Technology": ["QuixStreams"], + "Type": ["Code samples"], + "Pipeline Stage": ["Transformation"], + "Popular Subjects": ["MathWorks", "Simulink", "MATLAB", "Scientific Computing"] + }, + "shortDescription": "Transform Quix streams using Simulink", + "longDescription": "Run Simulink models on Quix as Python-compatible .whl packages", + "DefaultFile": "main.py", + "EntryPoint": "dockerfile", + "RunEntryPoint": "main.py", + "Variables": [ + { + "Name": "input", + "Type": "EnvironmentVariable", + "InputType": "InputTopic", + "Description": "Input topic.", + "DefaultValue": "input-topic", + "Required": true + }, + { + "Name": "output", + "Type": "EnvironmentVariable", + "InputType": "OutputTopic", + "Description": "Output topic.", + "DefaultValue": "output-topic", + "Required": true + } + ], + "DeploySettings": { + "DeploymentType": "Service", + "CpuMillicores": 2000, + "MemoryInMb": 4000, + "Replicas": 1, + "PublicAccess": false, + "ValidateConnection": false + } +} \ No newline at end of file diff --git a/python/transformations/simulink-wheel/main.py b/python/transformations/simulink-wheel/main.py new file mode 100644 index 00000000..ce798fee --- /dev/null +++ b/python/transformations/simulink-wheel/main.py @@ -0,0 +1,54 @@ +from quixstreams import Application +import numpy as np +import os +import quixmatlab + +# Initiate quixmatlab +quixmatlab_client = quixmatlab.initialize() +print("Exported MATLAB functions:", dir(quixmatlab_client)) + +# Define matlab function call +def matlab_processing(row: dict): + # Prepare function inputs + x = row["x"] + y = row["y"] + theta = np.pi/4 # 45 degrees in radians + + # Call function here + input_matrix = np.array([[0, x, y, theta]]) + # print("Input", input_matrix) + output_matrix = quixmatlab_client.simulink_wrapper(input_matrix) + # print("Output", output_matrix) + + # Incorporating result to row + row["x_new"] = output_matrix[0][0] + row["y_new"] = output_matrix[0][1] + + +def main(): + # Setup necessary objects + app = Application( + consumer_group="CompilerSDKMatlab2024b_wheel_general", + auto_create_topics=True, + auto_offset_reset="earliest" + ) + input_topic = app.topic(name=os.environ["input"]) + output_topic = app.topic(name=os.environ["output"]) + sdf = app.dataframe(topic=input_topic) + + + # Do StreamingDataFrame operations/transformations here + sdf.print_table() + sdf = sdf.update(matlab_processing) + sdf.print_table() + + # Finish off by writing to the final result to the output topic + sdf.to_topic(output_topic) + + # With our pipeline defined, now run the Application + app.run() + + +# It is recommended to execute Applications under a conditional main +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/python/transformations/simulink-wheel/quixmatlab_r2024b-24.2-py3-none-any.whl b/python/transformations/simulink-wheel/quixmatlab_r2024b-24.2-py3-none-any.whl new file mode 100755 index 00000000..f6e1ac25 Binary files /dev/null and b/python/transformations/simulink-wheel/quixmatlab_r2024b-24.2-py3-none-any.whl differ diff --git a/python/transformations/simulink-wheel/requirements.txt b/python/transformations/simulink-wheel/requirements.txt new file mode 100644 index 00000000..e828cf97 --- /dev/null +++ b/python/transformations/simulink-wheel/requirements.txt @@ -0,0 +1,6 @@ +quixstreams==3.17 +numpy +python-dotenv +setuptools +wheel +./quixmatlab_r2024b-24.2-py3-none-any.whl \ No newline at end of file