Update error messages for SparkConnect linter (#2348)

## Changes
Update phrasing in linter messages to be more clear. Replace `UC Shared
clusters` with `Unity Catalog clusters in Shared access mode`

Author: vsevolodstep-db

src/databricks/labs/ucx/source_code/linters/spark_connect.py

@@ -19,7 +19,11 @@ class SharedClusterMatcher:
     session_state: CurrentSessionState
 
     def _cluster_type_str(self) -> str:
-        return 'UC Shared Clusters' if not self.session_state.is_serverless else 'Serverless Compute'
+        return (
+            'Unity Catalog clusters in Shared access mode'
+            if not self.session_state.is_serverless
+            else 'Serverless Compute'
+        )
 
     @abstractmethod
     def lint(self, node: NodeNG) -> Iterator[Advice]:

tests/unit/source_code/linters/test_spark_connect.py

@@ -23,7 +23,7 @@ def test_jvm_access_match_shared(session_state):
     expected = [
         Failure(
             code="jvm-access-in-shared-clusters",
-            message='Cannot access Spark Driver JVM on UC Shared Clusters',
+            message='Cannot access Spark Driver JVM on Unity Catalog clusters in Shared access mode',
             start_line=2,
             start_col=0,
             end_line=2,
@@ -65,31 +65,31 @@ def test_rdd_context_match_shared(session_state):
     expected = [
         Failure(
             code="rdd-in-shared-clusters",
-            message='RDD APIs are not supported on UC Shared Clusters. Rewrite it using DataFrame API',
+            message='RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Rewrite it using DataFrame API',
             start_line=1,
             start_col=7,
             end_line=1,
             end_col=32,
         ),
         Failure(
             code="rdd-in-shared-clusters",
-            message='RDD APIs are not supported on UC Shared Clusters. Rewrite it using DataFrame API',
+            message='RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Rewrite it using DataFrame API',
             start_line=2,
             start_col=29,
             end_line=2,
             end_col=42,
         ),
         Failure(
             code='legacy-context-in-shared-clusters',
-            message='sc is not supported on UC Shared Clusters. Rewrite it using spark',
+            message='sc is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark',
             start_line=1,
             start_col=7,
             end_line=1,
             end_col=21,
         ),
         Failure(
             code="legacy-context-in-shared-clusters",
-            message='sc is not supported on UC Shared Clusters. Rewrite it using spark',
+            message='sc is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark',
             start_line=2,
             start_col=29,
             end_line=2,
@@ -152,7 +152,7 @@ def test_rdd_map_partitions(session_state):
     expected = [
         Failure(
             code="rdd-in-shared-clusters",
-            message='RDD APIs are not supported on UC Shared Clusters. Use mapInArrow() or Pandas UDFs instead',
+            message='RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Use mapInArrow() or Pandas UDFs instead',
             start_line=2,
             start_col=0,
             end_line=2,
@@ -169,7 +169,7 @@ def test_conf_shared(session_state):
     assert [
         Failure(
             code='legacy-context-in-shared-clusters',
-            message='sparkContext and getConf are not supported on UC Shared Clusters. Rewrite it using spark.conf',
+            message='sparkContext and getConf are not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark.conf',
             start_line=0,
             start_col=0,
             end_line=0,
@@ -211,7 +211,7 @@ def test_logging_shared(session_state):
     assert [
         Failure(
             code='spark-logging-in-shared-clusters',
-            message='Cannot set Spark log level directly from code on UC Shared Clusters. '
+            message='Cannot set Spark log level directly from code on Unity Catalog clusters in Shared access mode. '
             'Remove the call and set the cluster spark conf \'spark.log.level\' instead',
             start_line=1,
             start_col=0,
@@ -220,15 +220,17 @@ def test_logging_shared(session_state):
         ),
         Failure(
             code='spark-logging-in-shared-clusters',
-            message='Cannot access Spark Driver JVM logger on UC Shared Clusters. ' 'Use logging.getLogger() instead',
+            message='Cannot access Spark Driver JVM logger on Unity Catalog clusters in Shared access mode. '
+            'Use logging.getLogger() instead',
             start_line=4,
             start_col=14,
             end_line=4,
             end_col=38,
         ),
         Failure(
             code='spark-logging-in-shared-clusters',
-            message='Cannot access Spark Driver JVM logger on UC Shared Clusters. ' 'Use logging.getLogger() instead',
+            message='Cannot access Spark Driver JVM logger on Unity Catalog clusters in Shared access mode. '
+            'Use logging.getLogger() instead',
             start_line=6,
             start_col=0,
             end_line=6,

tests/unit/source_code/samples/functional/spark-connect/catalog-api_13_3.py

@@ -1,7 +1,7 @@
 # ucx[session-state] {"dbr_version": [13, 3], "data_security_mode": "USER_ISOLATION"}
-# ucx[catalog-api-in-shared-clusters:+1:0:+1:13] spark.catalog functions require DBR 14.3 LTS or above on UC Shared Clusters
+# ucx[catalog-api-in-shared-clusters:+1:0:+1:13] spark.catalog functions require DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode
 spark.catalog.tableExists("table")
-# ucx[catalog-api-in-shared-clusters:+1:0:+1:13] spark.catalog functions require DBR 14.3 LTS or above on UC Shared Clusters
+# ucx[catalog-api-in-shared-clusters:+1:0:+1:13] spark.catalog functions require DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode
 spark.catalog.listDatabases()
 
 

tests/unit/source_code/samples/functional/spark-connect/command-context.py

@@ -1,7 +1,7 @@
 # ucx[session-state] {"data_security_mode": "USER_ISOLATION"}
-# ucx[to-json-in-shared-clusters:+1:6:+1:80] toJson() is not available on UC Shared Clusters. Use toSafeJson() on DBR 13.3 LTS or above to get a subset of command context information.
+# ucx[to-json-in-shared-clusters:+1:6:+1:80] toJson() is not available on Unity Catalog clusters in Shared access mode. Use toSafeJson() on DBR 13.3 LTS or above to get a subset of command context information.
 print(dbutils.notebook.entry_point.getDbutils().notebook().getContext().toJson())
 dbutils.notebook.entry_point.getDbutils().notebook().getContext().toSafeJson()
 notebook = dbutils.notebook.entry_point.getDbutils().notebook()
-# ucx[to-json-in-shared-clusters:+1:0:+1:30] toJson() is not available on UC Shared Clusters. Use toSafeJson() on DBR 13.3 LTS or above to get a subset of command context information.
+# ucx[to-json-in-shared-clusters:+1:0:+1:30] toJson() is not available on Unity Catalog clusters in Shared access mode. Use toSafeJson() on DBR 13.3 LTS or above to get a subset of command context information.
 notebook.getContext().toJson()

tests/unit/source_code/samples/functional/spark-connect/jvm-access.py

@@ -1,7 +1,7 @@
 # ucx[session-state] {"data_security_mode": "USER_ISOLATION"}
 spark.range(10).collect()
-# ucx[jvm-access-in-shared-clusters:+1:0:+1:18] Cannot access Spark Driver JVM on UC Shared Clusters
+# ucx[jvm-access-in-shared-clusters:+1:0:+1:18] Cannot access Spark Driver JVM on Unity Catalog clusters in Shared access mode
 spark._jspark._jvm.com.my.custom.Name()
 
-# ucx[jvm-access-in-shared-clusters:+1:0:+1:18] Cannot access Spark Driver JVM on UC Shared Clusters
+# ucx[jvm-access-in-shared-clusters:+1:0:+1:18] Cannot access Spark Driver JVM on Unity Catalog clusters in Shared access mode
 spark._jspark._jvm.com.my.custom.Name()

tests/unit/source_code/samples/functional/spark-connect/python-udfs_13_3.py

@@ -8,7 +8,7 @@ def slen(s):
     return len(s)
 
 
-# ucx[python-udf-in-shared-clusters:+1:1:+1:37] Arrow UDFs require DBR 14.3 LTS or above on UC Shared Clusters
+# ucx[python-udf-in-shared-clusters:+1:1:+1:37] Arrow UDFs require DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode
 @udf(returnType='int', useArrow=True)
 def arrow_slen(s):
     return len(s)
@@ -18,7 +18,7 @@ def arrow_slen(s):
 df.select(slen("name"), arrow_slen("name")).show()
 
 slen1 = udf(lambda s: len(s), returnType='int')
-# ucx[python-udf-in-shared-clusters:+1:14:+1:68] Arrow UDFs require DBR 14.3 LTS or above on UC Shared Clusters
+# ucx[python-udf-in-shared-clusters:+1:14:+1:68] Arrow UDFs require DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode
 arrow_slen1 = udf(lambda s: len(s), returnType='int', useArrow=True)
 
 df = spark.createDataFrame([(1, "John Doe", 21)], ("id", "name", "age"))
@@ -33,7 +33,7 @@ def subtract_mean(pdf: pd.DataFrame) -> pd.DataFrame:
     return pdf.assign(v=v - v.mean())
 
 
-# ucx[python-udf-in-shared-clusters:+1:0:+1:73] applyInPandas require DBR 14.3 LTS or above on UC Shared Clusters
+# ucx[python-udf-in-shared-clusters:+1:0:+1:73] applyInPandas require DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode
 df.groupby("id").applyInPandas(subtract_mean, schema="id long, v double").show()
 
 
@@ -43,11 +43,11 @@ def eval(self, start: int, end: int):
             yield (num, num * num)
 
 
-# ucx[python-udf-in-shared-clusters:+1:13:+1:69] udtf require DBR 14.3 LTS or above on UC Shared Clusters
+# ucx[python-udf-in-shared-clusters:+1:13:+1:69] udtf require DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode
 square_num = udtf(SquareNumbers, returnType="num: int, squared: int")
 square_num(lit(1), lit(3)).show()
 
 from pyspark.sql.types import IntegerType
 
-# ucx[python-udf-in-shared-clusters:+1:0:+1:73] Cannot register Java UDF from Python code on UC Shared Clusters. Use a %scala cell to register the Scala UDF using spark.udf.register.
+# ucx[python-udf-in-shared-clusters:+1:0:+1:73] Cannot register Java UDF from Python code on Unity Catalog clusters in Shared access mode. Use a %scala cell to register the Scala UDF using spark.udf.register.
 spark.udf.registerJavaFunction("func", "org.example.func", IntegerType())

tests/unit/source_code/samples/functional/spark-connect/python-udfs_14_3.py

@@ -35,5 +35,5 @@ def subtract_mean(pdf: pd.DataFrame) -> pd.DataFrame:
 
 from pyspark.sql.types import IntegerType
 
-# ucx[python-udf-in-shared-clusters:+1:0:+1:73] Cannot register Java UDF from Python code on UC Shared Clusters. Use a %scala cell to register the Scala UDF using spark.udf.register.
+# ucx[python-udf-in-shared-clusters:+1:0:+1:73] Cannot register Java UDF from Python code on Unity Catalog clusters in Shared access mode. Use a %scala cell to register the Scala UDF using spark.udf.register.
 spark.udf.registerJavaFunction("func", "org.example.func", IntegerType())

tests/unit/source_code/samples/functional/spark-connect/rdd-apis.py

@@ -1,12 +1,12 @@
 # ucx[session-state] {"data_security_mode": "USER_ISOLATION"}
 df = spark.createDataFrame([])
-# ucx[rdd-in-shared-clusters:+1:0:+1:27] RDD APIs are not supported on UC Shared Clusters. Use mapInArrow() or Pandas UDFs instead
+# ucx[rdd-in-shared-clusters:+1:0:+1:27] RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Use mapInArrow() or Pandas UDFs instead
 df.rdd.mapPartitions(myUdf)
 
-# ucx[rdd-in-shared-clusters:+2:7:+2:32] RDD APIs are not supported on UC Shared Clusters. Rewrite it using DataFrame API
-# ucx[legacy-context-in-shared-clusters:+1:7:+1:21] sc is not supported on UC Shared Clusters. Rewrite it using spark
+# ucx[rdd-in-shared-clusters:+2:7:+2:32] RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Rewrite it using DataFrame API
+# ucx[legacy-context-in-shared-clusters:+1:7:+1:21] sc is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark
 rdd1 = sc.parallelize([1, 2, 3])
 
-# ucx[rdd-in-shared-clusters:+2:29:+2:42] RDD APIs are not supported on UC Shared Clusters. Rewrite it using DataFrame API
-# ucx[legacy-context-in-shared-clusters:+1:29:+1:40] sc is not supported on UC Shared Clusters. Rewrite it using spark
+# ucx[rdd-in-shared-clusters:+2:29:+2:42] RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Rewrite it using DataFrame API
+# ucx[legacy-context-in-shared-clusters:+1:29:+1:40] sc is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark
 rdd2 = spark.createDataFrame(sc.emptyRDD(), schema)

tests/unit/source_code/samples/functional/spark-connect/spark-logging.py

@@ -1,16 +1,16 @@
 # ucx[session-state] {"data_security_mode": "USER_ISOLATION"}
-# ucx[legacy-context-in-shared-clusters:+2:0:+2:14] sc is not supported on UC Shared Clusters. Rewrite it using spark
-# ucx[spark-logging-in-shared-clusters:+1:0:+1:22] Cannot set Spark log level directly from code on UC Shared Clusters. Remove the call and set the cluster spark conf 'spark.log.level' instead
+# ucx[legacy-context-in-shared-clusters:+2:0:+2:14] sc is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark
+# ucx[spark-logging-in-shared-clusters:+1:0:+1:22] Cannot set Spark log level directly from code on Unity Catalog clusters in Shared access mode. Remove the call and set the cluster spark conf 'spark.log.level' instead
 sc.setLogLevel("INFO")
 setLogLevel("WARN")
 
-# ucx[jvm-access-in-shared-clusters:+3:14:+3:21] Cannot access Spark Driver JVM on UC Shared Clusters
-# ucx[legacy-context-in-shared-clusters:+2:14:+2:21] sc is not supported on UC Shared Clusters. Rewrite it using spark
-# ucx[spark-logging-in-shared-clusters:+1:14:+1:38] Cannot access Spark Driver JVM logger on UC Shared Clusters. Use logging.getLogger() instead
+# ucx[jvm-access-in-shared-clusters:+3:14:+3:21] Cannot access Spark Driver JVM on Unity Catalog clusters in Shared access mode
+# ucx[legacy-context-in-shared-clusters:+2:14:+2:21] sc is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark
+# ucx[spark-logging-in-shared-clusters:+1:14:+1:38] Cannot access Spark Driver JVM logger on Unity Catalog clusters in Shared access mode. Use logging.getLogger() instead
 log4jLogger = sc._jvm.org.apache.log4j
 LOGGER = log4jLogger.LogManager.getLogger(__name__)
 
-# ucx[jvm-access-in-shared-clusters:+3:0:+3:7] Cannot access Spark Driver JVM on UC Shared Clusters
-# ucx[legacy-context-in-shared-clusters:+2:0:+2:7] sc is not supported on UC Shared Clusters. Rewrite it using spark
-# ucx[spark-logging-in-shared-clusters:+1:0:+1:24] Cannot access Spark Driver JVM logger on UC Shared Clusters. Use logging.getLogger() instead
+# ucx[jvm-access-in-shared-clusters:+3:0:+3:7] Cannot access Spark Driver JVM on Unity Catalog clusters in Shared access mode
+# ucx[legacy-context-in-shared-clusters:+2:0:+2:7] sc is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark
+# ucx[spark-logging-in-shared-clusters:+1:0:+1:24] Cannot access Spark Driver JVM logger on Unity Catalog clusters in Shared access mode. Use logging.getLogger() instead
 sc._jvm.org.apache.log4j.LogManager.getLogger(__name__).info("test")

tests/unit/source_code/test_lsp_plugin.py

@@ -53,14 +53,14 @@ def test_pylsp_lint(workspace, config):
             'range': {'start': {'line': 0, 'character': 0}, 'end': {'line': 0, 'character': 11}},
             'code': 'legacy-context-in-shared-clusters',
             'source': 'databricks.labs.ucx',
-            'message': 'sc is not supported on UC Shared Clusters. Rewrite it using spark',
+            'message': 'sc is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark',
             'severity': 1,
             'tags': [],
         },
         {
             'range': {'end': {'character': 35, 'line': 1}, 'start': {'character': 0, 'line': 1}},
             'code': 'python-udf-in-shared-clusters',
-            'message': 'applyInPandas require DBR 14.3 LTS or above on UC Shared Clusters',
+            'message': 'applyInPandas require DBR 14.3 LTS or above on Unity Catalog clusters in Shared access mode',
             'severity': 1,
             'source': 'databricks.labs.ucx',
             'tags': [],
@@ -69,7 +69,7 @@ def test_pylsp_lint(workspace, config):
             'range': {'start': {'line': 0, 'character': 0}, 'end': {'line': 0, 'character': 13}},
             'code': 'rdd-in-shared-clusters',
             'source': 'databricks.labs.ucx',
-            'message': 'RDD APIs are not supported on UC Shared Clusters. Rewrite it using DataFrame API',
+            'message': 'RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Rewrite it using DataFrame API',
             'severity': 1,
             'tags': [],
         },
@@ -93,15 +93,15 @@ def test_pylsp_lint_no_dbr_version(workspace, config):
             'range': {'start': {'line': 0, 'character': 0}, 'end': {'line': 0, 'character': 11}},
             'code': 'legacy-context-in-shared-clusters',
             'source': 'databricks.labs.ucx',
-            'message': 'sc is not supported on UC Shared Clusters. Rewrite it using spark',
+            'message': 'sc is not supported on Unity Catalog clusters in Shared access mode. Rewrite it using spark',
             'severity': 1,
             'tags': [],
         },
         {
             'range': {'start': {'line': 0, 'character': 0}, 'end': {'line': 0, 'character': 13}},
             'code': 'rdd-in-shared-clusters',
             'source': 'databricks.labs.ucx',
-            'message': 'RDD APIs are not supported on UC Shared Clusters. Rewrite it using DataFrame API',
+            'message': 'RDD APIs are not supported on Unity Catalog clusters in Shared access mode. Rewrite it using DataFrame API',
             'severity': 1,
             'tags': [],
         },