Merge branch 'master' into master

Author: mhamilton723

README.md

@@ -11,10 +11,10 @@ SynapseML requires Scala 2.12, Spark 3.4+, and Python 3.8+.
 | Topics  | Links                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
 | :------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | Build   | [![Build Status](https://msdata.visualstudio.com/A365/_apis/build/status/microsoft.SynapseML?branchName=master)](https://msdata.visualstudio.com/A365/_build/latest?definitionId=17563&branchName=master) [![codecov](https://codecov.io/gh/Microsoft/SynapseML/branch/master/graph/badge.svg)](https://codecov.io/gh/Microsoft/SynapseML) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)                     |
-| Version | [![Version](https://img.shields.io/badge/version-1.0.10-blue)](https://github.com/Microsoft/SynapseML/releases) [![Release Notes](https://img.shields.io/badge/release-notes-blue)](https://github.com/Microsoft/SynapseML/releases) [![Snapshot Version](https://mmlspark.blob.core.windows.net/icons/badges/master_version3.svg)](#sbt)                                                                                                                                       |
-| Docs    | [![Website](https://img.shields.io/badge/SynapseML-Website-blue)](https://aka.ms/spark) [![Scala Docs](https://img.shields.io/static/v1?label=api%20docs&message=scala&color=blue&logo=scala)](https://mmlspark.blob.core.windows.net/docs/1.0.10/scala/index.html#package) [![PySpark Docs](https://img.shields.io/static/v1?label=api%20docs&message=python&color=blue&logo=python)](https://mmlspark.blob.core.windows.net/docs/1.0.10/pyspark/index.html) [![Academic Paper](https://img.shields.io/badge/academic-paper-7fdcf7)](https://arxiv.org/abs/1810.08744) |
+| Version | [![Version](https://img.shields.io/badge/version-1.0.11-blue)](https://github.com/Microsoft/SynapseML/releases) [![Release Notes](https://img.shields.io/badge/release-notes-blue)](https://github.com/Microsoft/SynapseML/releases) [![Snapshot Version](https://mmlspark.blob.core.windows.net/icons/badges/master_version3.svg)](#sbt)                                                                                                                                       |
+| Docs    | [![Website](https://img.shields.io/badge/SynapseML-Website-blue)](https://aka.ms/spark) [![Scala Docs](https://img.shields.io/static/v1?label=api%20docs&message=scala&color=blue&logo=scala)](https://mmlspark.blob.core.windows.net/docs/1.0.11/scala/index.html#package) [![PySpark Docs](https://img.shields.io/static/v1?label=api%20docs&message=python&color=blue&logo=python)](https://mmlspark.blob.core.windows.net/docs/1.0.11/pyspark/index.html) [![Academic Paper](https://img.shields.io/badge/academic-paper-7fdcf7)](https://arxiv.org/abs/1810.08744) |
 | Support | [![Gitter](https://badges.gitter.im/Microsoft/MMLSpark.svg)](https://gitter.im/Microsoft/MMLSpark?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Mail](https://img.shields.io/badge/mail-synapseml--support-brightgreen)](mailto:synapseml-support@microsoft.com)                                                                                                                                                                                                  |
-| Binder  | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/microsoft/SynapseML/v1.0.10?labpath=notebooks%2Ffeatures)                                                                                                                                                                                                                                                                                                                                           |
+| Binder  | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/microsoft/SynapseML/v1.0.11?labpath=notebooks%2Ffeatures)                                                                                                                                                                                                                                                                                                                                           |
 | Usage | [![Downloads](https://static.pepy.tech/badge/synapseml)](https://pepy.tech/project/synapseml) |
 <!-- markdownlint-disable MD033 -->
 <details open>
@@ -119,7 +119,7 @@ In Azure Synapse notebooks please place the following in the first cell of your
 {
   "name": "synapseml",
   "conf": {
-      "spark.jars.packages": "com.microsoft.azure:synapseml_2.12:1.0.10",
+      "spark.jars.packages": "com.microsoft.azure:synapseml_2.12:1.0.11",
       "spark.jars.repositories": "https://mmlspark.azureedge.net/maven",
       "spark.jars.excludes": "org.scala-lang:scala-reflect,org.apache.spark:spark-tags_2.12,org.scalactic:scalactic_2.12,org.scalatest:scalatest_2.12,com.fasterxml.jackson.core:jackson-databind",
       "spark.yarn.user.classpath.first": "true",
@@ -155,15 +155,15 @@ cloud](http://community.cloud.databricks.com), create a new [library from Maven
 coordinates](https://docs.databricks.com/user-guide/libraries.html#libraries-from-maven-pypi-or-spark-packages)
 in your workspace.
 
-For the coordinates use: `com.microsoft.azure:synapseml_2.12:1.0.10`
+For the coordinates use: `com.microsoft.azure:synapseml_2.12:1.0.11`
 with the resolver: `https://mmlspark.azureedge.net/maven`. Ensure this library is
 attached to your target cluster(s).
 
 Finally, ensure that your Spark cluster has at least Spark 3.2 and Scala 2.12. If you encounter Netty dependency issues please use DBR 10.1.
 
 You can use SynapseML in both your Scala and PySpark notebooks. To get started with our example notebooks import the following databricks archive:
 
-`https://mmlspark.blob.core.windows.net/dbcs/SynapseMLExamplesv1.0.10.dbc`
+`https://mmlspark.blob.core.windows.net/dbcs/SynapseMLExamplesv1.0.11.dbc`
 
 ### Python Standalone
 
@@ -174,7 +174,7 @@ the above example, or from python:
 ```python
 import pyspark
 spark = pyspark.sql.SparkSession.builder.appName("MyApp") \
-            .config("spark.jars.packages", "com.microsoft.azure:synapseml_2.12:1.0.10") \
+            .config("spark.jars.packages", "com.microsoft.azure:synapseml_2.12:1.0.11") \
             .getOrCreate()
 import synapse.ml
 ```
@@ -185,9 +185,9 @@ SynapseML can be conveniently installed on existing Spark clusters via the
 `--packages` option, examples:
 
 ```bash
-spark-shell --packages com.microsoft.azure:synapseml_2.12:1.0.10
-pyspark --packages com.microsoft.azure:synapseml_2.12:1.0.10
-spark-submit --packages com.microsoft.azure:synapseml_2.12:1.0.10 MyApp.jar
+spark-shell --packages com.microsoft.azure:synapseml_2.12:1.0.11
+pyspark --packages com.microsoft.azure:synapseml_2.12:1.0.11
+spark-submit --packages com.microsoft.azure:synapseml_2.12:1.0.11 MyApp.jar
 ```
 
 ### SBT
@@ -196,7 +196,7 @@ If you are building a Spark application in Scala, add the following lines to
 your `build.sbt`:
 
 ```scala
-libraryDependencies += "com.microsoft.azure" % "synapseml_2.12" % "1.0.10"
+libraryDependencies += "com.microsoft.azure" % "synapseml_2.12" % "1.0.11"
 ```
 
 ### Apache Livy and HDInsight
@@ -210,7 +210,7 @@ Excluding certain packages from the library may be necessary due to current issu
 {
     "name": "synapseml",
     "conf": {
-        "spark.jars.packages": "com.microsoft.azure:synapseml_2.12:1.0.10",
+        "spark.jars.packages": "com.microsoft.azure:synapseml_2.12:1.0.11",
         "spark.jars.excludes": "org.scala-lang:scala-reflect,org.apache.spark:spark-tags_2.12,org.scalactic:scalactic_2.12,org.scalatest:scalatest_2.12,com.fasterxml.jackson.core:jackson-databind"
     }
 }

core/src/main/python/synapse/ml/core/logging/SynapseMLLogger.py

@@ -6,6 +6,7 @@
 from synapse.ml.core.platform.Platform import (
     running_on_synapse_internal,
     running_on_synapse,
+    running_on_fabric_python,
 )
 from pyspark.sql.dataframe import DataFrame
 from pyspark import SparkContext
@@ -82,7 +83,7 @@ def get_required_log_fields(
 
     @classmethod
     def _get_environment_logger(cls, log_level: int) -> logging.Logger:
-        if running_on_synapse_internal():
+        if running_on_synapse_internal() or running_on_fabric_python():
             from synapse.ml.pymds.synapse_logger import get_mds_logger
 
             return get_mds_logger(

core/src/main/python/synapse/ml/core/platform/Platform.py

@@ -5,11 +5,13 @@
 
 PLATFORM_SYNAPSE_INTERNAL = "synapse_internal"
 PLATFORM_SYNAPSE = "synapse"
+PLATFORM_FABRIC_PYTHON = "fabric_python_env"
 PLATFORM_BINDER = "binder"
 PLATFORM_DATABRICKS = "databricks"
 PLATFORM_UNKNOWN = "unknown"
 SECRET_STORE = "mmlspark-build-keys"
 SYNAPSE_PROJECT_NAME = "Microsoft.ProjectArcadia"
+FABRIC_URL = "fabric.microsoft.com"
 
 
 def current_platform():
@@ -26,6 +28,8 @@ def current_platform():
         return PLATFORM_DATABRICKS
     elif os.environ.get("BINDER_LAUNCH_HOST", None) is not None:
         return PLATFORM_BINDER
+    elif FABRIC_URL in os.environ.get("MSNOTEBOOKUTILS_SPARK_TRIDENT_PBIHOST", ""):
+        return PLATFORM_FABRIC_PYTHON
     else:
         return PLATFORM_UNKNOWN
 
@@ -46,6 +50,10 @@ def running_on_databricks():
     return current_platform() is PLATFORM_DATABRICKS
 
 
+def running_on_fabric_python():
+    return current_platform() is PLATFORM_FABRIC_PYTHON
+
+
 def find_secret(secret_name, keyvault):
     try:
         if running_on_synapse():

core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksCPUTests.scala

@@ -5,14 +5,13 @@ package com.microsoft.azure.synapse.ml.nbtest
 
 import com.microsoft.azure.synapse.ml.nbtest.DatabricksUtilities._
 
-import scala.collection.mutable.ListBuffer
 import scala.language.existentials
 
 class DatabricksCPUTests extends DatabricksTestHelper {
 
   val clusterId: String = createClusterInPool(ClusterName, AdbRuntime, NumWorkers, PoolId, memory = Some("7g"))
 
-  databricksTestHelper(clusterId, Libraries, CPUNotebooks)
+  databricksTestHelper(clusterId, Libraries, CPUNotebooks, 5)
 
   protected override def afterAll(): Unit = {
     afterAllHelper(clusterId, ClusterName)

core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksGPUTests.scala

@@ -3,18 +3,13 @@
 
 package com.microsoft.azure.synapse.ml.nbtest
 
-import com.microsoft.azure.synapse.ml.build.BuildInfo
-import com.microsoft.azure.synapse.ml.core.env.FileUtilities
 import com.microsoft.azure.synapse.ml.nbtest.DatabricksUtilities._
 
-import java.io.File
-import scala.collection.mutable.ListBuffer
-
 class DatabricksGPUTests extends DatabricksTestHelper {
 
   val clusterId: String = createClusterInPool(GPUClusterName, AdbGpuRuntime, 2, GpuPoolId)
 
-  databricksTestHelper(clusterId, GPULibraries, GPUNotebooks, 1)
+  databricksTestHelper(clusterId, GPULibraries, GPUNotebooks, 1, List())
 
   protected override def afterAll(): Unit = {
     afterAllHelper(clusterId, GPUClusterName)

core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksRapidsTests.scala

@@ -5,18 +5,11 @@ package com.microsoft.azure.synapse.ml.nbtest
 
 import com.microsoft.azure.synapse.ml.nbtest.DatabricksUtilities._
 
-import com.microsoft.azure.synapse.ml.build.BuildInfo
-import com.microsoft.azure.synapse.ml.core.env.FileUtilities
-import com.microsoft.azure.synapse.ml.nbtest.DatabricksUtilities._
-
-import java.io.File
-import scala.collection.mutable.ListBuffer
-
 class DatabricksRapidsTests extends DatabricksTestHelper {
 
   val clusterId: String = createClusterInPool(GPUClusterName, AdbGpuRuntime, 1, GpuPoolId, RapidsInitScripts)
 
-  databricksTestHelper(clusterId, GPULibraries, RapidsNotebooks)
+  databricksTestHelper(clusterId, GPULibraries, RapidsNotebooks, 4)
 
   protected override def afterAll(): Unit = {
     afterAllHelper(clusterId, RapidsClusterName)

core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala

@@ -21,11 +21,12 @@ import spray.json.{JsArray, JsObject, JsValue, _}
 import java.io.{File, FileInputStream}
 import java.time.LocalDateTime
 import java.util.concurrent.{Executors, TimeUnit, TimeoutException}
-import scala.collection.immutable.Map
 import scala.collection.mutable
 import scala.concurrent.duration.Duration
 import scala.concurrent.{Await, ExecutionContext, Future, blocking}
 import scala.util.Random
+import com.microsoft.azure.synapse.ml.io.http.RESTHelpers.retry
+
 
 object DatabricksUtilities {
 
@@ -434,7 +435,8 @@ abstract class DatabricksTestHelper extends TestBase {
   def databricksTestHelper(clusterId: String,
                            libraries: String,
                            notebooks: Seq[File],
-                           maxConcurrency: Int = 8): Unit = {
+                           maxConcurrency: Int,
+                           retries: List[Int] = List(1000 * 15)): Unit = {
 
     println("Checking if cluster is active")
     tryWithRetries(Seq.fill(60 * 20)(1000).toArray) { () =>
@@ -455,7 +457,9 @@ abstract class DatabricksTestHelper extends TestBase {
 
     val futures = notebooks.map { notebook =>
       Future {
-        runNotebook(clusterId, notebook)
+        retry(retries, { () =>
+          runNotebook(clusterId, notebook)
+        })
       }
     }
     futures.zip(notebooks).foreach { case (f, nb) =>