|
| 1 | +--- |
| 2 | +title: Enable automatic differentiation of C++ STL concurrency primitives in Clad |
| 3 | +layout: gsoc_proposal |
| 4 | +project: Clad |
| 5 | +year: 2025 |
| 6 | +difficulty: medium |
| 7 | +duration: 350 |
| 8 | +mentor_avail: June-October |
| 9 | +organization: |
| 10 | + - CompRes |
| 11 | +--- |
| 12 | + |
| 13 | +## Description |
| 14 | + |
| 15 | +Clad is an automatic differentiation (AD) clang plugin for C++. Given a C++ source code of a mathematical function, it can automatically generate C++ code for computing derivatives of the function. This project focuses on enabling automatic differentiation of codes that utilise C++ concurrency features such as `std::thread`, `std::mutex`, atomic operations and more. This will allow users to fully utilize their CPU resources. |
| 16 | + |
| 17 | +## Expected Results |
| 18 | + |
| 19 | +* Explore C++ concurrency primitives and prepare a report detailing the associated challenges involved and the features that can be feasibly supported within the given timeframe. |
| 20 | +* Add concurrency primitives support in Clad’s forward-mode automatic differentiation. |
| 21 | +* Add concurrency primitives support in Clad’s reverse-mode automatic differentiation. |
| 22 | +* Add proper tests and documentation. |
| 23 | +* Present the work at the relevant meetings and conferences. |
| 24 | + |
| 25 | +An example demonstrating the use of differentiation of codes utilizing parallelization primitives: |
| 26 | + |
| 27 | +``` |
| 28 | +#include <cmath> |
| 29 | +#include <iostream> |
| 30 | +#include <mutex> |
| 31 | +#include <numeric> |
| 32 | +#include <thread> |
| 33 | +#include <vector> |
| 34 | +#include "clad/Differentiator/Differentiator.h" |
| 35 | +
|
| 36 | +using VectorD = std::vector<double>; |
| 37 | +using MatrixD = std::vector<VectorD>; |
| 38 | +
|
| 39 | +std::mutex m; |
| 40 | +
|
| 41 | +VectorD operator*(const VectorD &l, const VectorD &r) { |
| 42 | + VectorD v(l.size()); |
| 43 | + for (std::size_t i = 0; i < l.size(); ++i) |
| 44 | + v[i] = l[i] * r[i]; |
| 45 | + return v; |
| 46 | +} |
| 47 | +
|
| 48 | +double dot(const VectorD &v1, const VectorD &v2) { |
| 49 | + VectorD v = v1 * v2; |
| 50 | + return std::accumulate(v.begin(), v.end(), 0.0); |
| 51 | +} |
| 52 | +
|
| 53 | +double activation_fn(double z) { return 1 / (1 + std::exp(-z)); } |
| 54 | +
|
| 55 | +double compute_loss(double y, double y_estimate) { |
| 56 | + return -(y * std::log(y_estimate) + (1 - y) * std::log(1 - y_estimate)); |
| 57 | +} |
| 58 | +
|
| 59 | +void compute_and_add_loss(VectorD x, double y, const VectorD &weights, double b, |
| 60 | + double &loss) { |
| 61 | + double z = dot(x, weights) + b; |
| 62 | + double y_estimate = activation_fn(z); |
| 63 | + std::lock_guard<std::mutex> guard(m); |
| 64 | + loss += compute_loss(y, y_estimate); |
| 65 | +} |
| 66 | +
|
| 67 | +/// Compute total loss associated with a single neural neural-network. |
| 68 | +/// y_estimate = activation_fn(dot(X[i], weights) + b) |
| 69 | +/// Loss of a training data point = - (y_actual * std::log(y_estimate) + (1 - y_actual) * std::log(1 - y_estimate)) |
| 70 | +/// total loss: summation of loss for all the data points |
| 71 | +double compute_total_loss(const MatrixD &X, const VectorD &Y, |
| 72 | + const VectorD &weights, double b) { |
| 73 | + double loss = 0; |
| 74 | + const std::size_t num_of_threads = std::thread::hardware_concurrency(); |
| 75 | + std::vector<std::thread> threads(num_of_threads); |
| 76 | + int thread_id = 0; |
| 77 | + for (std::size_t i = 0; i < X.size(); ++i) { |
| 78 | + if (threads[thread_id].joinable()) |
| 79 | + threads[thread_id].join(); |
| 80 | + threads[thread_id] = |
| 81 | + std::thread(compute_and_add_loss, std::cref(X[i]), Y[i], |
| 82 | + std::cref(weights), b, std::ref(loss)); |
| 83 | + thread_id = (thread_id + 1) % num_of_threads; |
| 84 | + } |
| 85 | + for (std::size_t i = 0; i < num_of_threads; ++i) { |
| 86 | + if (threads[i].joinable()) |
| 87 | + threads[i].join(); |
| 88 | + } |
| 89 | +
|
| 90 | + return loss; |
| 91 | +} |
| 92 | +
|
| 93 | +int main() { |
| 94 | + auto loss_grad = clad::gradient(compute_total_loss); |
| 95 | + // Fill the values as required! |
| 96 | + MatrixD X; |
| 97 | + VectorD Y; |
| 98 | + VectorD weights; |
| 99 | + double b; |
| 100 | +
|
| 101 | + // derivatives |
| 102 | + // Zero the derivative variables and make them of the same dimension as the |
| 103 | + // corresponding primal values. |
| 104 | + MatrixD d_X; |
| 105 | + VectorD d_Y; |
| 106 | + VectorD d_weights; |
| 107 | + double d_b = 0; |
| 108 | +
|
| 109 | + loss_grad.execute(X, Y, weights, b, &d_X, &d_Y, &d_weights, &d_b); |
| 110 | +
|
| 111 | + std::cout << "dLossFn/dW[2]: " << d_weights[2] << "\n"; // Partial derivative of the loss function w.r.t weight[2] |
| 112 | + std::cout << "dLossFn/db: " << d_b << "\n"; // Partial derivative of the loss function w.r.t b |
| 113 | +} |
| 114 | +``` |
| 115 | + |
| 116 | +## Requirements |
| 117 | + |
| 118 | +* Automatic differentiation |
| 119 | +* Parallel programming |
| 120 | +* Reasonable expertise in C++ programming |
| 121 | + |
| 122 | +## Mentors |
| 123 | +* **[Vassil Vassilev ](mailto:[email protected])** |
| 124 | +* [David Lange ](mailto:[email protected]) |
| 125 | + |
| 126 | +## Links |
| 127 | +* [Repo](https://github.com/vgvassilev/clad) |
0 commit comments