diff --git a/scripts/algorithms/L2_SVM_.daph b/scripts/algorithms/L2_SVM_.daph new file mode 100755 index 000000000..9e901fe0b --- /dev/null +++ b/scripts/algorithms/L2_SVM_.daph @@ -0,0 +1,124 @@ +#------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +#------------------------------------------------------------- +This script has been manually translated from Apache SystemDS +# Original file: systemds/scripts/builtin/l2svm.dml +# This builting function implements binary-class Support Vector Machine (SVM) +# with squared slack variables (l2 regularization). +# +# INPUT: +# X: Feature matrix (m x n) +# Y: Label vector (m x 1), assumed binary (-1/+1 or 1/2 encoding) +# intercept: Boolean, add bias column (default: false) +# epsilon: Early stopping threshold (default: 0.001) +# reg: L2 regularization parameter (default: 1.0) +# maxIterations: Max outer iterations (default: 100) +# maxii: Max line search iterations (default: 20) +# verbose: Boolean, print progress (default: false) +# columnId: Optional class ID for verbose mode (default: -1) +# OUTPUT: +# model: Trained weight vector (n x 1, or n+1 with intercept) +def l2svm(X: Matrix, Y: Matrix, intercept: Boolean = false, epsilon: Float = 0.001, + reg: Float = 1.0, maxIterations: Int = 100, maxii: Int = 20, + verbose: Boolean = false, columnId: Int = -1): Matrix = { + + # Ensure valid input dimensions + if (X.numRows < 2) + throw "L2SVM: At least 2 rows are required to learn a classifier." + if (epsilon < 0.0) + throw "L2SVM: Tolerance (epsilon) must be non-negative." + if (reg < 0.0) + throw "L2SVM: Regularization constant (reg) must be non-negative." + if (maxIterations < 1) + throw "L2SVM: Maximum iterations should be a positive integer." + if (Y.numCols < 1) + throw "L2SVM: Label vector must have at least one column." + + # Convert labels to -1/+1 if needed + val minY = Y.min() + val maxY = Y.max() + val numMin = (Y == minY).sum() + val numMax = (Y == maxY).sum() + + if (numMin + numMax != Y.numRows) + println("L2SVM: WARNING invalid labels in Y: " + numMin + " " + numMax) + + val normalizedY = if (minY != -1 || maxY != 1) + (2.0 / (maxY - minY)) * Y - (minY + maxY) / (maxY - minY) + else + Y + + # Add bias column if intercept is enabled + val XWithBias = if (intercept) + X.appendCol(Matrix.ones(X.numRows, 1)) + else + X + + var w = Matrix.zeros(XWithBias.numCols, 1) + var Xw = Matrix.zeros(XWithBias.numRows, 1) + var gOld = XWithBias.transpose() %*% normalizedY + var s = gOld + var iter = 0 + var continueTraining = true + + while (continueTraining && iter < maxIterations) { + # Line search initialization + var stepSize = 0.0 + val Xd = XWithBias %*% s + val wd = reg * (w * s).sum() + val dd = reg * (s * s).sum() + var continueLineSearch = true + var innerIter = 0 + + while (continueLineSearch && innerIter < maxii) { + val tempXw = Xw + stepSize * Xd + val out = Matrix.maximum(1.0 - (normalizedY * tempXw), 0.0) + val sv = (out > 0.0) + val g = wd + stepSize * dd - (out * (normalizedY * Xd)).sum() + val h = dd + ((Xd * sv) * Xd).sum() + stepSize = stepSize - g / h + continueLineSearch = (g * g / h >= epsilon) + innerIter += 1 + } + + # Update weights + w = w + stepSize * s + Xw = Xw + stepSize * Xd + + val out = Matrix.maximum(1.0 - (normalizedY * Xw), 0.0) + val obj = 0.5 * (out * out).sum() + 0.5 * reg * (w * w).sum() + val gNew = XWithBias.transpose() %*% (out * normalizedY) - reg * w + + if (verbose) { + val colStr = if (columnId != -1) s"-- MSVM class=$columnId: " else "" + println(s"$colStr Iter: $iter InnerIter: $innerIter --- Obj: $obj") + } + + # Check stopping condition + val temp = (s * gOld).sum() + continueTraining = (stepSize * temp >= epsilon * obj && (s * s).sum() != 0.0) + + # Non-linear conjugate gradient step + val beta = (gNew * gNew).sum() / (gOld * gOld).sum() + s = beta * s + gNew + gOld = gNew + iter += 1 + } + + return w +} + diff --git a/scripts/algorithms/SVM_Multiclass_.daph b/scripts/algorithms/SVM_Multiclass_.daph new file mode 100644 index 000000000..43fdeed55 --- /dev/null +++ b/scripts/algorithms/SVM_Multiclass_.daph @@ -0,0 +1 @@ +#------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # #------------------------------------------------------------- # This script has been manually translated from Apache SystemDS (https://github.com/apache/systemds). # Original file: scripts/builtin/msvm.dml # This script implements a multi-class Support Vector Machine (SVM) # with squared slack variables. The trained model comprises #classes # one-against-the-rest binary-class l2svm classification models. # # INPUT: #------------------------------------------------------------------------------- # X Feature matrix X (shape: m x n) # Y Label vector y of class labels (shape: m x 1), # where max(Y) is assumed to be the number of classes # intercept Indicator if a bias column should be added to X and the model # epsilon Tolerance for early termination if the reduction of objective # function is less than epsilon times the initial objective # reg Regularization parameter (lambda) for L2 regularization # maxIterations Maximum number of conjugate gradient (outer l2svm) iterations # verbose Indicator if training details should be printed # ------------------------------------------------------------------------------ # # OUTPUT: #------------------------------------------------------------------------------- # model Trained model/weights (shape: n x max(Y), w/ intercept: n+1) #------------------------------------------------------------------------------- def msvm(X:matrix, Y:matrix, intercept:bool /*= false*/, epsilon:double /*= 0.001*/, reg:double /*= 1.0*/, maxIterations:int /*= 100*/, verbose:bool /*= false*/) -> matrix { if( min(Y) < 0 ) stop("MSVM: Invalid Y input, containing negative values"); if(verbose) print("Running Multiclass-SVM"); # Robustness for datasets with missing values (causing NaN gradients) numNaNs = sum(isNan(X)); if(numNaNs > 0) { print("msvm: matrix X contains "+numNaNs+" missing values, replacing with 0."); X = replace(X, Double.NaN, 0.0); } # Append bias term if intercept is requested if(intercept) { ones = fill(1.0, nrow(X), 1); X = cbind(X, ones); } if(ncol(Y) > 1) Y = as.f64(idxMax(t(Y), 0)); # Assuming number of classes to be max contained in Y numClasses = max(Y); w = fill(0.0, ncol(X), numClasses); # Loop over each class for(class in 1:numClasses) { # Extract the binary labels for the current class and convert to -1/+1 Y_local = 2 * (Y == class) - 1; # Train the L2-SVM model for the current class nnzY = sum(Y == class); if(nnzY > 0) { w[,class - 1] = l2svm(X, Y_local, false, epsilon, reg, maxIterations, verbose); } else { w[,class - 1] = fill(-Infinity, ncol(X), 1); } } return w; } \ No newline at end of file diff --git a/scripts/algorithms/SVM_Predict_.daph b/scripts/algorithms/SVM_Predict_.daph new file mode 100644 index 000000000..a455b6741 --- /dev/null +++ b/scripts/algorithms/SVM_Predict_.daph @@ -0,0 +1 @@ +#------------------------------------------------------------- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS # OF ANY KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # #------------------------------------------------------------- # This script has been manually translated from Apache SystemDS (https://github.com/apache/systemds). # Original file: scripts/builtin/msvmPredict.dml # This script helps in applying a trained MSVM model # # INPUT: #------------------------------------------------------------------------------ # X Feature matrix X to classify # W Matrix of trained variables (weights) #------------------------------------------------------------------------------ # # OUTPUT: #------------------------------------------------------------------------------ # YRaw Classification Labels Raw (unprocessed, might contain 1's and -1's) # Y Classification Labels processed (final prediction: maxed to ones/zeros) #------------------------------------------------------------------------------ m_msvmPredict = function(X:matrix, W:matrix) return (YRaw:matrix, Y:matrix) { # Handle missing values (NaNs) numNaNs = sum(isnan(X)); if (numNaNs > 0) { print("msvmPredict: matrix X contains " + numNaNs + " missing values, replacing with 0."); X = replace(X, Double.NaN, 0.0); } # Ensure compatibility between the feature matrix X and weight matrix W if (ncol(X) != nrow(W)) { if (ncol(X) + 1 != nrow(W)) { stop("MSVM Predict: Invalid shape of W [" + nrow(W) + "," + ncol(W) + "] or X [" + nrow(X) + "," + ncol(X) + "]"); } # Perform matrix multiplication and add bias term YRaw = X %*% W[1:ncol(X),] + W[ncol(X)+1,]; Y = as.f64(idxMax(t(YRaw), 0)); } else { # Perform matrix multiplication without bias term YRaw = X %*% W; Y = as.f64(idxMax(t(YRaw), 0)); } } \ No newline at end of file diff --git a/scripts/algorithms/mSVM_Test_.daph b/scripts/algorithms/mSVM_Test_.daph new file mode 100755 index 000000000..42031e210 --- /dev/null +++ b/scripts/algorithms/mSVM_Test_.daph @@ -0,0 +1,41 @@ +# Test Case for Multiclass SVM with User-Defined Parameters +def test_multiclass_svm(num_samples: Integer = 100, + num_features: Integer = 4, + num_classes: Integer = 3, + max_iterations: Integer = 100, + reg_param: Double = 0.1, + verbose: Boolean = false) -> None +{ + # Generate synthetic data + print("Generating synthetic dataset...") + X = rand(num_samples, num_features, 0.0, 1.0, 1) # Feature matrix + Y = matrix(0, rows=num_samples, cols=1) + + for i in 1:num_classes { + # Assign classes in a round-robin fashion + for j in (i-1):(num_samples-1):num_classes { + Y[j, 0] = i + } + } + + print("Dataset generated:") + print("Features: ", X) + print("Labels: ", Y) + + # Train the Multiclass SVM model + print("Training Multiclass SVM with user-defined parameters...") + model = multiclass_svm(X, Y, reg_param, max_iterations, verbose) + + # Predict using the trained model + print("Predicting using the trained model...") + predictions = predict_svm(X, model) + + # Calculate accuracy + correct = sum(predictions == Y) + accuracy = correct / num_samples * 100.0 + print("Accuracy: ", accuracy, "%") + + +} + +