#22 add scalafmt to repo (#23)

* #22 adding scalafmt with a basic config targetting Scala 2.11

* #22 adding optional scalafmt CI check

* #22 post-CQC changes in scalafmt conf we were discussing

* #22 scalafmt config - moving closer to our codebase (without these, function definition formatting was very different)

* #22 scalafmt will check only files that were changed in comparison to master branch

* #22 post-review changes in CI file

* #22 scalafmt: `origin/master` doesn't work in CI check, renaming to `master`

* #22 scalafmt: `origin/master` and `master` doesn't work in CI check, trying `remotes/origin/master`

* #22 scalafmt: downgrading scalafmt version, there might be a bug in the 2.4.6

See: scalameta/sbt-scalafmt#205

* #22 scalafmt: replicing master branch with the actual base branch

* #22 scalafmt: tmp commit (I want to see how CI check behaves and what's in the log)

* #22: checking out the branch, not the PR remote (head would be in a detached state)

This is because for scalafmt we need to find the 'base' branch - which is master in this case, and compare the list of changed files against it, in order to check the formatting against such newly changed files only. Also, 'origin/' isn't needed because the 'base' branch will be 'origin/master', not just 'master' - see file build.sbt, line starting with `scalafmtFilter.withRank` and the expression starting with `git show-branch -a`.

* #22 requested in CQC session, for preventing this to be evaluated when it's not needed

* #22 post-review changes, improving readability

Co-authored-by: Ladislav Sulak <[email protected]>

Author: lsulak

.github/workflows/format_check.yml

@@ -0,0 +1,42 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: FormatCheck
+
+on:
+  pull_request:
+    branches: [ master, develop ]
+    types: [ assigned, opened, synchronize, reopened, labeled ]
+
+jobs:
+  scalafmt:
+    runs-on: ubuntu-latest
+    name: Scalafmt Check
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+
+      - name: Setup Scala
+        uses: olafurpg/setup-scala@v10
+        with:
+          java-version: "[email protected]"
+
+      - name: Run scalafmt And Print Diff
+        continue-on-error: true
+        run: sbt scalafmt scalafmtSbt && git diff --exit-code

.scalafmt.conf

@@ -0,0 +1,20 @@
+version = "3.5.3"
+runner.dialect = scala211
+
+maxColumn = 120
+
+align.preset = some
+align.multiline = false
+
+indent.main = 2
+indent.defnSite = 2
+
+lineEndings = unix
+
+docstrings.blankFirstLine = yes
+docstrings.style = AsteriskSpace
+docstrings.wrap = no
+docstrings.removeEmpty = true
+
+align.openParenDefnSite = true
+align.openParenCallSite = true

build.sbt

@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+import sys.process._
 
 ThisBuild / name := "spark-data-standardization"
 ThisBuild / organization := "za.co.absa"
@@ -27,11 +27,11 @@ ThisBuild / scalaVersion := scala211
 
 def sparkVersion(scalaVersion: String): String = if (scalaVersion==scala212) "3.2.1" else "2.4.7"
 
-def sparkFastTestsVersion(scalaVersion: String): String = if (scalaVersion==scala212) "1.1.0" else "0.23.0"
+def sparkFastTestsVersion(scalaVersion: String): String = if (scalaVersion == scala212) "1.1.0" else "0.23.0"
 
-libraryDependencies ++=  List(
+libraryDependencies ++= List(
   "org.apache.spark" %% "spark-core" % sparkVersion(scalaVersion.value) % "provided",
-  "org.apache.spark" %% "spark-sql" % sparkVersion(scalaVersion.value)  % "provided",
+  "org.apache.spark" %% "spark-sql" % sparkVersion(scalaVersion.value) % "provided",
   "za.co.absa" %% "spark-commons" % "0.2.0",
   "com.github.mrpowers" %% "spark-fast-tests" % sparkFastTestsVersion(scalaVersion.value) % Test,
   "org.scalatest" %% "scalatest" % "3.2.2" % Test,
@@ -47,6 +47,19 @@ ThisBuild / printSparkScalaVersion := {
 
 Test / parallelExecution := false
 
+// Only apply scalafmt to files that differ from master (i.e. files changed in the feature branch or so),
+// not on the whole repository.
+lazy val currBranchName = "git branch --show-current".!!.trim
+lazy val baseBranchName = (
+  "git show-branch -a"
+    #| raw"grep \*"
+    #| s"grep -v $currBranchName"
+    #| "head -n1"
+    #| raw"sed s/.*\[\(.*\)\].*/\1/"
+).!!.trim
+
+scalafmtFilter.withRank(KeyRanks.Invisible) := s"diff-ref=${baseBranchName}"
+
 // licenceHeader check:
 ThisBuild / organizationName := "ABSA Group Limited"
 ThisBuild / startYear := Some(2021)

project/plugins.sbt

@@ -1 +1,2 @@
 addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.7.0")
+addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.5")

src/main/scala/za/co/absa/standardization/Standardization.scala

@@ -21,6 +21,7 @@ import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types.{StructType, _}
 import org.apache.spark.sql.{Column, DataFrame, SparkSession}
 import org.slf4j.{Logger, LoggerFactory}
+
 import za.co.absa.spark.commons.implicits.StructTypeImplicits.StructTypeEnhancements
 import za.co.absa.standardization.config.{DefaultStandardizationConfig, StandardizationConfig}
 import za.co.absa.standardization.stages.{SchemaChecker, TypeParser}