Merge branch 'master' into support-new-datatypes

# Conflicts:
#	CHANGELOG.md
#	realm/realm-library/src/androidTest/java/io/realm/RealmMigrationTests.java
#	realm/realm-library/src/main/cpp/realm-core
#	version.txt

Author: cmelchior

CHANGELOG.md

@@ -1,6 +1,8 @@
-## 10.5.0 (YYYY-MM-DD)
+## 10.6.0-BETA.1 (YYYY-MM-DD)
 
 ### Breaking Changes
+* [RealmApp] Sync protocol version increased to 3. This version adds support for the new data types introduced in file format version 21.
+* File format version bumped to 21. In this version we support new basic datatypes `UUID` and `RealmAny`, as well as `RealmSet` and `RealmMap` collections with string-based keys (i.e. `RealmDictionary`).
 * Queries no longer do nullability checks on non-nullable fields, so using `null` as an argument will not throw an `IllegalArgumentException`.
 * String query filters `contains`, `beginsWith`, `endsWith`, and `like`, now throw a null pointer exception on null values.
 * The query builder no longer throw `IllegalStateException` but `IllegalArgumentException`.
@@ -34,12 +36,40 @@
 * Classes names "class_class_..." were not handled correctly when doing queries ([#4480](https://github.com/realm/realm-core/issues/4480))
 * Fix collection notification reporting for modifications. This could be observed by receiving the wrong indices of modifications on sorted or distinct results, or notification blocks sometimes not being called when only modifications have occurred. ([#4573](https://github.com/realm/realm-core/pull/4573) since v6).
 
-### Breaking changes
-* Sync protocol version increased to 3. This version adds support for the new data types introduced in file format version 21.
-* File format version bumped to 21. In this version we support new basic datatypes `UUID` and `RealmAny`, as well as `RealmSet` and `RealmMap` collections with string-based keys (i.e. `RealmDictionary`).
+### Compatibility
+* File format: Generates Realms with format v21. Unsynced Realms will be upgraded from Realm Java 2.0 and later. Synced Realms can only be read and upgraded if created with Realm Java v10.0.0-BETA.1.
+* APIs are backwards compatible with all previous release of realm-java in the 10.6.y series.
+* Realm Studio 10.0.0 or above is required to open Realms created by this version.
 
 ### Internal
-* Updated to Realm Core commit: <TBD>.
+* Updated to Realm Core 11.0.0-beta.4, commit: d50aef63a8aaf435e3afed82b589b47d8e1ab1ab.
+
+
+## 10.5.0 (2021-05-07)
+
+### Breaking Changes
+* [RealmApp] `SyncSession.State.WaitingForAccessToken` has been added. It represents the local access token not longer being valid, but is automatically being refreshed.
+
+### Enhancements
+* We now make a backup of a Realm file prior to any file format upgrade. The backup is retained for 3 months. Backups from before a file format upgrade allows for better analysis of any upgrade failure. We also restore a backup, if a) an attempt is made to open a realm file whith a "future" file format and b) a backup file exist that fits the current file format. The backup file is placed next to the real Realm file and is named `<fileName>.v<realmFileFormatVersion>.backup.realm`.
+* The error message when the intial steps of opening a Realm file fails is now more descriptive.
+
+### Fixes
+* [RealmApp] Client Reset errors now correctly forward the server error message. (Issue [#7363](https://github.com/realm/realm-java/issues/7363), since 10.0.0)
+* [RealmApp] All `AppException`s now correctly report the error message through `RuntimeException.getMessage()` instead of only through `AppException.getErrorMessage()`. 
+* [RealmApp] Proactively check the expiry time on the access token and refresh it before attempting to initiate a sync session. This prevents some error logs from appearing on the client such as: "ERROR: Connection[1]: Websocket: Expected HTTP response 101 Switching Protocols, but received: HTTP/1.1 401 Unauthorized" (RCORE-473, since v10.0.0).
+* Fix name aliasing not working in sort/distinct clauses of raw string predicates.
+* Fix collection notification reporting for modifications. This could be observed by receiving the wrong indices of modifications on sorted or distinct results, or notification sometimes not being called when only modifications have occured. (since v7.0.0).
+* Make conversion of Decimal128 to/from string work for numbers with more than 19 significant digits. (#4548)
+
+### Compatibility
+* File format: Generates Realms with format v20. Unsynced Realms will be upgraded from Realm Java 2.0 and later. Synced Realms can only be read and upgraded if created with Realm Java v10.0.0-BETA.1.
+* APIs are backwards compatible with all previous release of realm-java in the 10.x.y series.
+* Realm Studio 10.0.0 or above is required to open Realms created by this version.
+
+### Internal
+* Updated to Realm Core 10.7.1, commit 5043c25e1d8f5971002e0fec85dea5ea3d7eb3d7.
+
 
 ## 10.4.0 (2021-03-26)
 

Jenkinsfile

@@ -341,19 +341,22 @@ def runBuild(buildFlags, instrumentationTestTarget) {
 def runPublish() {
   stage('Publish Release') {
     withCredentials([
+            [$class: 'StringBinding', credentialsId: 'maven-central-java-ring-file', variable: 'SIGN_KEY'],
+            [$class: 'StringBinding', credentialsId: 'maven-central-java-ring-file-password', variable: 'SIGN_KEY_PASSWORD'],
             [$class: 'StringBinding', credentialsId: 'slack-webhook-java-ci-channel', variable: 'SLACK_URL_CI'],
             [$class: 'StringBinding', credentialsId: 'slack-webhook-releases-channel', variable: 'SLACK_URL_RELEASE'],
             [$class: 'UsernamePasswordMultiBinding', credentialsId: 'maven-central-credentials', passwordVariable: 'MAVEN_CENTRAL_PASSWORD', usernameVariable: 'MAVEN_CENTRAL_USER'],
             [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'DOCS_S3_ACCESS_KEY', credentialsId: 'mongodb-realm-docs-s3', secretKeyVariable: 'DOCS_S3_SECRET_KEY'],
             [$class: 'AmazonWebServicesCredentialsBinding', accessKeyVariable: 'REALM_S3_ACCESS_KEY', credentialsId: 'realm-s3', secretKeyVariable: 'REALM_S3_SECRET_KEY']
     ]) {
+      // TODO Make sure that buildFlags and signingFlags are unified across builds
       sh """
         set +x
         sh tools/publish_release.sh '$MAVEN_CENTRAL_USER' '$MAVEN_CENTRAL_PASSWORD' \
         '$REALM_S3_ACCESS_KEY' '$REALM_S3_SECRET_KEY' \
         '$DOCS_S3_ACCESS_KEY' '$DOCS_S3_SECRET_KEY' \
-        '$SLACK_URL_RELEASE' \
-        '$SLACK_URL_CI'
+        '$SLACK_URL_RELEASE' '$SLACK_URL_CI' \
+        '-PsignBuild=true -PsignSecretRingFile="${SIGN_KEY}" -PsignPassword=${SIGN_KEY_PASSWORD} -PenableLTO=true -PbuildCore=true'
       """
     }
   }

README.md

@@ -19,7 +19,7 @@ See [Realm Kotlin](https://github.com/realm/realm-kotlin) for more information a
 
 ## Getting Started
 
-Please see the [detailed instructions in our docs](https://realm.io/docs/java/latest/#installation) to add Realm to your project.
+Please see the [detailed instructions in our docs](https://docs.mongodb.com/realm/sdk/android/install/) to add Realm to your project.
 
 ## Documentation
 

build.gradle

@@ -1,16 +1,15 @@
 buildscript {
+    def properties = new Properties()
+    properties.load(new FileInputStream("${projectDir}/dependencies.list"))
+
     repositories {
         jcenter()
-    }
-    dependencies {
-        classpath 'io.codearte.gradle.nexus:gradle-nexus-staging-plugin:0.30.0'
-        classpath 'ch.netzwerg:gradle-release-plugin:1.2.0'
+        maven {
+            url "https://plugins.gradle.org/m2/"
+        }
     }
 }
 
-apply plugin: 'ch.netzwerg.release'
-apply plugin: 'io.codearte.nexus-staging'
-
 def currentVersion = file("${projectDir}/version.txt").text.trim()
 
 // Find property in either System environment or Gradle properties.
@@ -292,7 +291,6 @@ task assemble {
 
 task distributionJniUnstrippedPackage(type:Zip) {
     description = 'Generate native libs package with debug symbols'
-    dependsOn assembleRealm
 
     group = 'Artifact'
     archiveName = "realm-java-jni-libs-unstripped-${currentVersion}.zip"
@@ -301,6 +299,15 @@ task distributionJniUnstrippedPackage(type:Zip) {
     from("realm/realm-library/build/outputs/jniLibs-unstripped") {
         include '**/*.so'
     }
+    doLast {
+        // Failsafe check, ensuring that we catch if the path ever changes, which it might since it is an
+        // implementation detail of the Android Gradle Plugin
+        def unstrippedDir = file("${rootDir}/realm/realm-library/build/outputs/jniLibs-unstripped")
+        print(unstrippedDir)
+        if (!unstrippedDir.exists() || !unstrippedDir.isDirectory() || unstrippedDir.listFiles().size() == 0) {
+            throw new GradleException("Could not locate unstripped binary files in: ${unstrippedDir.getPath()}")
+        }
+    }
 }
 
 task cleanRealm(type:GradleBuild) {
@@ -404,7 +411,7 @@ task mavenCentralRealm(type: GradleBuild) {
     description = 'Publish the Realm AAR and AP to Maven Central'
     group = 'Publishing'
     buildFile = file('realm/build.gradle')
-    tasks = ['publishAllPublicationsToMavenCentralRepository']
+    tasks = ['publishToSonatype']
     startParameter.projectProperties = gradle.startParameter.projectProperties
     configure copyProperties
 }
@@ -413,24 +420,27 @@ task mavenCentralAnnotations(type: GradleBuild) {
     description = 'Publish the Realm Annotations to Maven Central'
     group = 'Publishing'
     buildFile = file('realm-annotations/build.gradle')
+    tasks = ['publishToSonatype']
     startParameter.projectProperties = gradle.startParameter.projectProperties
-    tasks = ['publishAllPublicationsToMavenCentralRepository']
+    configure copyProperties
 }
 
 task mavenCentralGradlePlugin(type: GradleBuild) {
     description = 'Publish the Realm Gradle Plugin to Maven Central'
     group = 'Publishing'
     buildFile = file('gradle-plugin/build.gradle')
+    tasks = ['publishToSonatype']
     startParameter.projectProperties = gradle.startParameter.projectProperties
-    tasks = ['publishAllPublicationsToMavenCentralRepository']
+    configure copyProperties
 }
 
 task mavenCentralTransformer(type: GradleBuild) {
     description = 'Publish the Realm Transformer to Maven Central'
     group = 'Publishing'
     buildFile = file('realm-transformer/build.gradle')
+    tasks = ['publishToSonatype']
     startParameter.projectProperties = gradle.startParameter.projectProperties
-    tasks = ['publishAllPublicationsToMavenCentralRepository']
+    configure copyProperties
 }
 
 task mavenCentralUpload {
@@ -441,26 +451,3 @@ task mavenCentralUpload {
     dependsOn mavenCentralGradlePlugin
     dependsOn mavenCentralTransformer
 }
-
-apply from: "mavencentral-properties.gradle"
-
-nexusStaging {
-    // Lazily evaluate properties so they are only needed when actually used.
-    packageGroup = "io.realm"
-    stagingProfileId = rootProject.ext["sonatypeStagingProfileId"]
-    username = rootProject.ext["ossrhUsername"]
-    password = rootProject.ext["ossrhPassword"]
-    // Maven Central, at times, seems really slow to update the repository status,
-    // so setting the timeout to 20 minutes.
-    delayBetweenRetriesInMillis = 10000
-    numberOfRetries = 120
-}
-
-// This is just a placeholder for the release plugin
-task build {}
-
-release {
-    push = false
-    versionSuffix = '-SNAPSHOT'
-    tagPrefix = 'v'
-}

dependencies.list

@@ -17,6 +17,7 @@ gradle=6.5
 ndkVersion=22.0.7026061
 BUILD_INFO_EXTRACTOR_GRADLE=4.17.0
 GRADLE_BINTRAY_PLUGIN=1.8.5
+GRADLE_NEXUS_PLUGIN=1.0.0
 CMAKE=3.18.4
 
 # Bson dependency version

gradle-plugin/build.gradle

@@ -4,10 +4,14 @@ buildscript {
 
     repositories {
         jcenter()
+        maven {
+            url "https://plugins.gradle.org/m2/"
+        }
     }
     dependencies {
         classpath "org.jfrog.buildinfo:build-info-extractor-gradle:${properties.get('BUILD_INFO_EXTRACTOR_GRADLE')}"
         classpath "com.jfrog.bintray.gradle:gradle-bintray-plugin:${properties.get('GRADLE_BINTRAY_PLUGIN')}"
+        classpath "io.github.gradle-nexus:publish-plugin:${properties.get("GRADLE_NEXUS_PLUGIN")}"
     }
 }
 
@@ -84,6 +88,7 @@ sourceSets {
 
 compileJava.dependsOn generateVersionClass
 
+apply from: "${rootDir}/../mavencentral-publications.gradle"
 apply from: "${rootDir}/../mavencentral-publish.gradle"
 
 publishing {

library-build-transformer/build.gradle

@@ -9,10 +9,12 @@ buildscript {
     repositories {
         mavenCentral()
         jcenter()
+        maven {
+            url "https://plugins.gradle.org/m2/"
+        }
     }
     dependencies {
-        classpath "org.jfrog.buildinfo:build-info-extractor-gradle:${properties.get('BUILD_INFO_EXTRACTOR_GRADLE')}"
-        classpath "com.jfrog.bintray.gradle:gradle-bintray-plugin:${properties.get('GRADLE_BINTRAY_PLUGIN')}"
+        classpath "io.github.gradle-nexus:publish-plugin:${properties.get("GRADLE_NEXUS_PLUGIN")}"
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
     }
 }
@@ -23,8 +25,7 @@ version = file("${projectDir}/../version.txt").text.trim()
 apply plugin: 'kotlin'
 apply plugin: 'maven'
 apply plugin: 'maven-publish'
-apply plugin: 'com.jfrog.artifactory'
-apply plugin: 'com.jfrog.bintray'
+apply from: "${rootDir}/../mavencentral-publications.gradle"
 
 repositories {
     google()

mavencentral-publications.gradle

@@ -0,0 +1,106 @@
+// This script is responsible for setting up the publications, so they are ready to be published
+// to Maven Central and other repositories. Publishing to Maven Central is configured in
+// `mavencentral-publish.gradle`. They are in separate files as the nexus publish plugin used
+// to deploy to Maven Central can only be applied to top level projects.
+apply plugin: 'maven-publish'
+apply plugin: 'signing'
+apply from: buildscript.sourceFile.getParent() + "/" + "mavencentral-properties.gradle"
+
+// The publications doesn't know about our AAR dependencies, so we have to manually add them to the pom
+// Credit: http://stackoverflow.com/questions/24743562/gradle-not-including-dependencies-in-published-pom-xml
+def createPomDependencies(configurationNames) {
+    return {
+        def dependenciesNode = asNode().appendNode('dependencies')
+        configurationNames.each { configurationName ->
+            configurations[configurationName].allDependencies.each {
+                // Also ignore `realm-library` because that is used by the Kotlin Extensions as
+                // a project dependency, but it is being written to the POM file in a wrong way.
+                // Instead we just ignore it as the Gradle Plugin makes sure it is always present.
+                if (it.group != null && it.name != null && it.name != 'realm-library') {
+                    def dependencyNode = dependenciesNode.appendNode('dependency')
+                    dependencyNode.appendNode('groupId', it.group)
+                    dependencyNode.appendNode('artifactId', it.name)
+                    dependencyNode.appendNode('version', it.version)
+
+                    //If there are any exclusions in dependency
+                    if (it.excludeRules.size() > 0) {
+                        def exclusionsNode = dependencyNode.appendNode('exclusions')
+                        it.excludeRules.each { rule ->
+                            def exclusionNode = exclusionsNode.appendNode('exclusion')
+                            exclusionNode.appendNode('groupId', rule.group)
+                            exclusionNode.appendNode('artifactId', rule.module)
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+// TODO: Consider refactoring this. See https://github.com/realm/realm-java/pull/7327#discussion_r586281652
+def populatePom(publication, pomName, pomDescription, pomDependencies = null) {
+    publication.pom {
+        name = "${pomName}"
+        description = "${pomDescription}"
+        url = 'https://docs.mongodb.com/realm'
+        licenses {
+            license {
+                name = 'The Apache Software License, Version 2.0'
+                url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+                distribution = 'repo'
+            }
+        }
+        issueManagement {
+            system = 'github'
+            url = 'https://github.com/realm/realm-java/issues'
+        }
+        scm {
+            url = 'scm:https://github.com/realm/realm-java'
+            connection = 'scm:[email protected]:realm/realm-java.git'
+            developerConnection = 'scm:[email protected]:realm/realm-java.git'
+        }
+        developers {
+            developer {
+                name = 'Realm'
+                email = '[email protected]'
+                organization = 'MongoDB'
+                organizationUrl = 'https://www.mongodb.com'
+            }
+        }
+    }
+    if (pomDependencies) {
+        publication.pom.withXml(pomDependencies)
+    }
+}
+
+publishing {
+    repositories {
+        maven {
+            name = "MavenCentral"
+            def releasesRepoUrl = "https://oss.sonatype.org/service/local/staging/deploy/maven2/"
+            def snapshotsRepoUrl = "https://oss.sonatype.org/content/repositories/snapshots/"
+            url = version.endsWith('-SNAPSHOT') ? snapshotsRepoUrl : releasesRepoUrl
+            credentials {
+                username = rootProject.ext["ossrhUsername"]
+                password = rootProject.ext["ossrhPassword"]
+            }
+        }
+    }
+}
+
+signing {
+    required { signBuild }
+    def keyId = rootProject.ext["signing.keyId"]
+    def ringFile = rootProject.ext["signing.secretKeyRingFile"]
+    def password = rootProject.ext["signing.password"]
+    useInMemoryPgpKeys(keyId, ringFile, password)
+    sign publishing.publications
+}
+
+// Export methods so they are available in other gradle files
+// https://stackoverflow.com/questions/18715137/extract-common-methods-from-gradle-build-script
+ext {
+    createPomDependencies = this.&createPomDependencies
+    populatePom = this.&populatePom
+}
+