Merge branch 'master' into docs-fixes

Author: Jolanrensen

.editorconfig

@@ -1,16 +1,37 @@
 [*]
-charset=utf-8
-end_of_line=lf
-insert_final_newline=true
-indent_style=space
-indent_size=4
-max_line_length=120
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 4
+max_line_length = 120
 
 [*.json]
-indent_size=2
+indent_size = 2
 
-[*.yaml]
-indent_size=2
+[{*.yaml,*.yml}]
+indent_size = 2
 
 [*.ipynb]
-insert_final_newline=false
+insert_final_newline = false
+
+[*.{kt,kts}]
+ij_kotlin_code_style_defaults = KOTLIN_OFFICIAL
+
+#  Disable wildcard imports entirely
+ij_kotlin_name_count_to_use_star_import = 2147483647
+ij_kotlin_name_count_to_use_star_import_for_members = 2147483647
+ij_kotlin_packages_to_use_import_on_demand = unset
+
+ktlint_code_style = ktlint_official
+ktlint_experimental = enabled
+ktlint_standard_filename = disabled
+ktlint_standard_no-empty-first-line-in-class-body = disabled
+ktlint_class_signature_rule_force_multiline_when_parameter_count_greater_or_equal_than = 4
+ktlint_function_signature_rule_force_multiline_when_parameter_count_greater_or_equal_than = 4
+ktlint_standard_chain-method-continuation = disabled
+ktlint_ignore_back_ticked_identifier = true
+ktlint_standard_multiline-expression-wrapping = disabled
+
+[{*/build/**/*,**/*keywords*/**,**/*.Generated.kt,**/*$Extensions.kt}]
+ktlint = disabled

.github/workflows/generated-sources-master.yml

@@ -0,0 +1,33 @@
+name: Auto-commit generated code
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+
+      - name: Run Gradle task
+        run: ./gradlew :core:processKDocsMain
+
+      - name: Commit changes
+        run: |
+          git config --global user.name 'github-actions[bot]'
+          git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+          git add ./core/generated-sources
+          git diff --staged --quiet || git commit -m "Automated commit of generated code"
+          git push
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/generated-sources.yml

@@ -0,0 +1,92 @@
+name: Show generated code in PR
+
+on:
+  pull_request:
+    types:
+      - edited
+      - opened
+      - synchronize
+      - converted_to_draft
+      - ready_for_review
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '11'
+
+      - name: Configure Git User
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "GitHub Actions"
+
+      - name: Run Gradle task
+        run: ./gradlew :core:processKDocsMain
+
+      - name: Check for changes in generated sources
+        id: git-diff
+        run: echo "::set-output name=changed::$(if git diff --quiet './core/generated-sources'; then echo 'false'; else echo 'true'; fi)"
+
+      - name: Commit and push if changes
+        id: git-commit
+        if: steps.git-diff.outputs.changed == 'true'
+        run: |
+          git checkout -b generated-sources/docs-update-${{ github.run_number }}
+          git add './core/generated-sources'
+          git commit -m "Update generated sources with recent changes"
+          git push origin generated-sources/docs-update-${{ github.run_number }}
+          echo "::set-output name=commit::$(git rev-parse HEAD)"
+
+      - name: Remove old comments
+        uses: actions/github-script@v5
+        if: steps.git-diff.outputs.changed == 'true'
+        with:
+          # language=js
+          script: |
+            const issue_number = context.issue.number;
+            const {owner, repo} = context.repo;
+            
+            const comments = await github.rest.issues.listComments({
+                issue_number,
+                owner,
+                repo,
+            });
+            
+            const botComments = comments.data.filter(
+                (comment) => comment.user.login === 'github-actions[bot]'
+            );
+            
+            for (const comment of botComments) {
+                await github.rest.issues.deleteComment({
+                    comment_id: comment.id,
+                    owner,
+                    repo,
+                });
+            }
+
+      - name: Add comment to PR
+        uses: actions/github-script@v5
+        if: steps.git-diff.outputs.changed == 'true'
+        with:
+          # language=js
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: "Generated sources will be updated after merging this PR.\nPlease inspect the changes in [here](https://github.com/${{ github.repository }}/commit/${{ steps.git-commit.outputs.commit }}).",
+            });

.github/workflows/main.yml

@@ -33,15 +33,15 @@ jobs:
           apt-get update
           apt-get install -y zip
           mkdir -p artifacts
-      - name: Build docs and include sitemap.xml
+      - name: Build docs # and include sitemap.xml TODO: uncomment when sitemap.xml is available
         run: |
           export DISPLAY=:99
           Xvfb :99 &
           /opt/builder/bin/idea.sh helpbuilderinspect -source-dir . -product $PRODUCT --runner github -output-dir artifacts/ || true
           test -e artifacts/$ARTIFACT && echo $ARTIFACT exists
-          cp docs/StardustDocs/sitemap.xml artifacts/sitemap.xml
-          cd artifacts
-          zip -r $ARTIFACT sitemap.xml
+#          cp docs/StardustDocs/sitemap.xml artifacts/sitemap.xml
+#          cd artifacts
+#          zip -r $ARTIFACT sitemap.xml
         working-directory: ${{ github.workspace }}
       - name: Upload modified documentation artifact
         uses: actions/upload-artifact@v3

.gitignore

@@ -1,5 +1,6 @@
 # Default ignored files
 .idea
 .gradle
+.kotlin
 build
 .ipynb_checkpoints

CONTRIBUTING.md

@@ -33,7 +33,8 @@ so do familiarize yourself with the following guidelines.
     * PR should be linked with the issue,
       excluding minor documentation changes, adding unit tests, and fixing typos.
 * If you make any code changes:
-    * Follow the [Kotlin Coding Conventions](https://kotlinlang.org/docs/reference/coding-conventions.html).
+    * Follow the [Kotlin Coding Conventions](https://kotlinlang.org/docs/reference/coding-conventions.html). 
+      [Ktlint](https://pinterest.github.io/ktlint/latest/) can help here.
     * [Build the project](#building) to ensure it all works and passes the tests.
 * If you fix a bug:
     * Write the test that reproduces the bug.
@@ -51,18 +52,21 @@ so do familiarize yourself with the following guidelines.
 
 ## PR workflow
 
-0. The contributor builds the library locally and runs all unit tests via the Gradle task `dataframe:test`
-   (see the ["Building"](#building) chapter).
+0. The contributor builds the library locally and runs all unit tests via the Gradle task 
+   `dataframe:test -Pkotlin.dataframe.debug=true` (see the ["Building"](#building) chapter).
 1. The contributor submits the PR if the local build is successful and the tests are green.
-2. The reviewer put his name in the "Reviewers" section of the proposed PR at the start of the review process.
-3. The reviewer leaves the comments or marks the PR with the abbreviation "LGTM" (Looks good to me).
+2. The reviewer puts their name in the "Reviewers" section of the proposed PR at the start of the review process.
+3. The reviewer leaves comments or marks the PR with the abbreviation "LGTM" (Looks good to me).
 4. The contributor answers the comments or fixes the proposed PR.
 5. The reviewer marks the PR with the word "LGTM."
-6. The maintainer could suggest merging the master branch to the PR branch a few times due to changes in the `master` branch.
-7. The maintainer runs TC builds (unit tests and examples as integration tests).
-8. TC writes the result (passed or not passed) to the PR checks at the bottom of the proposed PR.
-9. If it is possible, maintainers share the details of the failed build with the contributor.
-10. Maintainer merges the PR if all checks are successful and there is no conflict with the master branch.
+6. The maintainer could suggest merging the `master` branch to the PR branch a few times due to changes in the `master` branch.
+7. If the PR influences generated code/samples, a bot will inform about this in the PR comments.
+8. The maintainer runs TeamCity builds (unit tests and examples as integration tests).
+9. TeamCity writes the result (passed or not passed) to the PR checks at the bottom of the proposed PR.
+10. If it is possible, maintainers share the details of the failed build with the contributor.
+11. The maintainer merges the PR if all checks are successful and there is no conflict with the `master` branch.
+12. The maintainer closes the PR and the issue linked to it.
+13. If the PR influences generated code, a bot will auto-commit the newly generated code into the `master` branch.
 
 ## How to fix an existing issue
 
@@ -81,15 +85,28 @@ so do familiarize yourself with the following guidelines.
 
 ## Environment requirements
 
-JDK >= 11 referred to by the `JAVA_HOME` environment variable.
+* JDK >= 11 referred to by the `JAVA_HOME` environment variable.
+
+  * Note, any version above 11 should work in theory, but JDK 11 is the only version we test with,
+  so it is the recommended version.
+
+* We recommend using [IntelliJ IDEA](https://www.jetbrains.com/idea/download/) as the IDE. This
+has the best support for Kotlin, compiler plugins, Gradle, and [Kotlin Notebook](https://kotlinlang.org/docs/kotlin-notebook-overview.html) of course.
+
+* We recommend using the [Ktlint plugin](https://plugins.jetbrains.com/plugin/15057-ktlint) for [IntelliJ IDEA](https://www.jetbrains.com/idea/download/).
+It is able to read the `.editorconfig` file and apply the same formatting rules as [Ktlint](https://pinterest.github.io/ktlint/latest/) in the CI.
+
+* Check out the [KDoc Preprocessor guide](KDOC_PREPROCESSING.md) to understand how to work with the KDoc preprocessor.
 
 ## Building
 
 This library is built with Gradle.
 
-* Run `./gradlew build` to build. It also runs all the tests.
+* Run `./gradlew build` to build. It also runs all the tests and checks the linter.
 * Run `./gradlew <module>:test` to test the module you are looking at to speed
   things up during development.
+* Make sure to pass the extra parameter `-Pkotlin.dataframe.debug=true` to enable debug mode. This flag will
+  make sure some extra checks are run, which are important but too heavy for production.
 
 You can import this project into IDEA, but you have to delegate the build actions
 to Gradle (in Preferences -> Build, Execution, Deployment -> Build Tools -> Gradle -> Runner)