Allow synchronization without Event data class (#128)

* UID handling: always process main event

- Modify `UidProcessor` to always use the UID from the main event and refactor the UID extraction logic into a companion object.
- Adjust `UidBuilder` to consistently use the UID from the main event.

* Update AndroidEventProcessor to ensure events have UIDs

- Add UidGenerator interface for generating new UIDs

* AndroidEventProcessor: return `MappingResult` with UID and generation status

* AndroidEventProcessor: also update SEQUENCE

* Update SEQUENCE and UID handling; tests

* Add comments

* Improve organizer handling in calendar sync

- Update OrganizerBuilder to handle ownerAccount more robustly
- Extract emailFromOrganizer function for better clarity
- Add logging for ignored organizers without email addresses

* KDoc

* KDoc

* Add `findEventAndExceptions` method to retrieve events and exceptions

* Add test

Author: rfc2822

lib/src/androidTest/kotlin/at/bitfire/synctools/storage/calendar/AndroidCalendarTest.kt

@@ -113,6 +113,26 @@ class AndroidCalendarTest {
         assertEntitiesEqual(entity, result, onlyFieldsInExpected = true)
     }
 
+    @Test
+    fun testFindEvent() {
+        // no result
+        assertNull(calendar.findEvent("${Events.DTSTART}=?", arrayOf(now.toString())))
+
+        // insert event
+        val entity = Entity(contentValuesOf(
+            Events.CALENDAR_ID to calendar.id,
+            Events.DTSTART to now,
+            Events.DTEND to now + 3600000,
+            Events.TITLE to "Some Event"
+        ))
+        calendar.addEvent(entity)
+
+        // not it finds a result
+        val result = calendar.findEvents("${Events.DTSTART}=?", arrayOf(now.toString()))
+        assertEquals(1, result.size)
+        assertEntitiesEqual(entity, result.first(), onlyFieldsInExpected = true)
+    }
+
     @Test
     fun testFindEvents() {
         calendar.addEvent(Entity(contentValuesOf(

lib/src/main/kotlin/at/bitfire/synctools/mapping/calendar/AndroidEventBuilder.kt

@@ -79,7 +79,7 @@ class AndroidEventBuilder(
         AvailabilityBuilder(),
         RecurrenceFieldsBuilder(),
         OriginalInstanceTimeBuilder(),
-        OrganizerBuilder(calendar.ownerAccount),
+        OrganizerBuilder(calendar.ownerAccount ?: calendar.account.name),
         UidBuilder(),
         // sub-rows (alphabetically, by class name)
         AttendeesBuilder(calendar),

lib/src/main/kotlin/at/bitfire/synctools/mapping/calendar/AndroidEventProcessor.kt

@@ -7,7 +7,10 @@
 package at.bitfire.synctools.mapping.calendar
 
 import android.content.Entity
+import android.provider.CalendarContract
 import android.provider.CalendarContract.Events
+import android.provider.CalendarContract.ExtendedProperties
+import androidx.annotation.VisibleForTesting
 import at.bitfire.synctools.icalendar.AssociatedEvents
 import at.bitfire.synctools.mapping.calendar.processor.AccessLevelProcessor
 import at.bitfire.synctools.mapping.calendar.processor.AndroidEventFieldProcessor
@@ -21,7 +24,6 @@ import at.bitfire.synctools.mapping.calendar.processor.EndTimeProcessor
 import at.bitfire.synctools.mapping.calendar.processor.LocationProcessor
 import at.bitfire.synctools.mapping.calendar.processor.OrganizerProcessor
 import at.bitfire.synctools.mapping.calendar.processor.OriginalInstanceTimeProcessor
-import at.bitfire.synctools.mapping.calendar.processor.ProdIdGenerator
 import at.bitfire.synctools.mapping.calendar.processor.RecurrenceFieldsProcessor
 import at.bitfire.synctools.mapping.calendar.processor.RemindersProcessor
 import at.bitfire.synctools.mapping.calendar.processor.SequenceProcessor
@@ -32,6 +34,7 @@ import at.bitfire.synctools.mapping.calendar.processor.UidProcessor
 import at.bitfire.synctools.mapping.calendar.processor.UnknownPropertiesProcessor
 import at.bitfire.synctools.mapping.calendar.processor.UrlProcessor
 import at.bitfire.synctools.storage.calendar.EventAndExceptions
+import at.bitfire.synctools.storage.calendar.EventsContract
 import net.fortuna.ical4j.model.DateList
 import net.fortuna.ical4j.model.Property
 import net.fortuna.ical4j.model.TimeZoneRegistryFactory
@@ -42,12 +45,13 @@ import net.fortuna.ical4j.model.property.RDate
 import net.fortuna.ical4j.model.property.RRule
 import net.fortuna.ical4j.model.property.RecurrenceId
 import java.util.LinkedList
+import java.util.UUID
 
 /**
  * Mapper from Android event main + data rows to [VEvent].
  *
- * @param accountName       account name (used to generate self-attendee)
- * @param prodIdGenerator   generator for `PRODID`
+ * @param accountName       account name (used to generate reminder email address)
+ * @param prodIdGenerator   generates the `PRODID` to use
  */
 class AndroidEventProcessor(
     accountName: String,
@@ -84,21 +88,55 @@ class AndroidEventProcessor(
     )
 
 
-    fun populate(eventAndExceptions: EventAndExceptions): AssociatedEvents {
-        // main event
-        val main = populateEvent(
+    /**
+     * Result of the [mapToVEvents] operation.
+     *
+     * @param associatedEvents  mapped events
+     * @param uid               UID of the mapped events
+     * @param generatedUid      whether [uid] was generated by [mapToVEvents] (*false*: `UID` was already present before mapping)
+     * @param updatedSequence   the new increased `SEQUENCE` of the main event (*null* if sequence was not increased by [mapToVEvents])
+     */
+    class MappingResult(
+        val associatedEvents: AssociatedEvents,
+        val uid: String,
+        val generatedUid: Boolean,
+        val updatedSequence: Int?
+    )
+
+    /**
+     * Maps an Android event with its exceptions to VEVENTs.
+     *
+     * VEVENTs must have a valid `UID`, so this method (or better to say, the [UidProcessor] that it calls)
+     * generates an UID, if necessary. If an `UID` was generated, it is noted in the result.
+     *
+     * This method also increases the `SEQUENCE`, if necessary, so that the mapped `SEQUENCE`
+     * can be different than the original `SEQUENCE` in [eventAndExceptions]. If the `SEQUENCE` was
+     * increased, it is noted in the result.
+     */
+    fun mapToVEvents(eventAndExceptions: EventAndExceptions): MappingResult {
+        // make sure that main event has a UID
+        var generatedUid = false
+        val uid = provideUid(eventAndExceptions.main) {
+            generatedUid = true
+            UUID.randomUUID().toString()
+        }
+
+        val updatedSequence = increaseSequence(eventAndExceptions.main)
+
+        // map main event
+        val main = mapEvent(
             entity = eventAndExceptions.main,
             main = eventAndExceptions.main
         )
 
-        // Add exceptions of recurring main event
+        // add exceptions of recurring main event
         val rRules = main.getProperties<RRule>(Property.RRULE)
         val rDates = main.getProperties<RDate>(Property.RDATE)
         val exceptions = LinkedList<VEvent>()
         if (rRules.isNotEmpty() || rDates.isNotEmpty()) {
             for (exception in eventAndExceptions.exceptions) {
                 // convert exception to Event
-                val exceptionEvent = populateEvent(
+                val exceptionEvent = mapEvent(
                     entity = exception,
                     main = eventAndExceptions.main
                 )
@@ -114,11 +152,17 @@ class AndroidEventProcessor(
             }
         }
 
-        return AssociatedEvents(
+        val mappedEvents = AssociatedEvents(
             main = main,
             exceptions = exceptions,
             prodId = generateProdId(eventAndExceptions.main)
         )
+        return MappingResult(
+            associatedEvents = mappedEvents,
+            uid = uid,
+            generatedUid = generatedUid,
+            updatedSequence = updatedSequence
+        )
     }
 
     private fun asExDate(entity: Entity, recurrenceId: RecurrenceId): ExDate {
@@ -145,6 +189,61 @@ class AndroidEventProcessor(
         return prodIdGenerator.generateProdId(packages)
     }
 
+    /**
+     * Increases the event's SEQUENCE, if necessary.
+     *
+     * @param main  event to be checked (**will be modified** when SEQUENCE needs to be increased)
+     *
+     * @return updated sequence (or *null* if sequence was not increased/modified)
+     */
+    @VisibleForTesting
+    internal fun increaseSequence(main: Entity): Int? {
+        val mainValues = main.entityValues
+        val currentSeq = mainValues.getAsInteger(EventsContract.COLUMN_SEQUENCE)
+
+        if (currentSeq == null) {
+            /* First upload, request to set to 0 in calendar provider after upload.
+            We can let it empty in the Entity because then no SEQUENCE property will be generated,
+            which is equal to SEQUENCE:0. */
+            return 0
+        }
+
+        val groupScheduled = main.subValues.any { it.uri == CalendarContract.Attendees.CONTENT_URI }
+        if (groupScheduled) {
+            /* Note: Events.IS_ORGANIZER is defined in CalendarDatabaseHelper.java as
+            COALESCE(Events.IS_ORGANIZER, Events.ORGANIZER = Calendars.OWNER_ACCOUNT), so it's non-null when it's
+            - either explicitly set for an event,
+            - or the event's ORGANIZER is the same as the calendar's OWNER_ACCOUNT. */
+            val weAreOrganizer = when (mainValues.getAsInteger(Events.IS_ORGANIZER)) {
+                null, 0 -> false
+                /* explicitly set to non-zero, or 1 by provider calculation */ else -> true
+            }
+
+            return if (weAreOrganizer) {
+                /* Upload of a group-scheduled event and we are the organizer, so we increase the SEQUENCE.
+                We also have to store it into the Entity so that the new value will be mapped. */
+                (currentSeq + 1).also { newSeq ->
+                    mainValues.put(EventsContract.COLUMN_SEQUENCE, newSeq)
+                }
+            } else
+                /* Upload of a group-scheduled event and we are not the organizer, so we don't increase the SEQUENCE. */
+                null
+
+        } else /* not group-scheduled */  {
+            return if (currentSeq == 0) {
+                /* The event was uploaded once and has SEQUENCE of 0 (which is mapped to an empty SEQUENCE property).
+                We don't need to increase the SEQUENCE because the event is not group-scheduled. */
+                null
+            } else {
+                /* Upload of a non-group-scheduled event where a SEQUENCE > 0 is present. Increase by one after upload.
+                We also have to store it into the Entity so that the new value will be mapped. */
+                (currentSeq + 1).also { newSeq ->
+                    mainValues.put(EventsContract.COLUMN_SEQUENCE, newSeq)
+                }
+            }
+        }
+    }
+
     /**
      * Reads data of an event from the calendar provider, i.e. converts the [entity] values into a [VEvent].
      *
@@ -153,13 +252,51 @@ class AndroidEventProcessor(
      *
      * @return generated data object
      */
-    private fun populateEvent(entity: Entity, main: Entity): VEvent {
-        val vEvent = VEvent()
+    private fun mapEvent(entity: Entity, main: Entity): VEvent {
+        // initialization adds DTSTAMP
+        val vEvent = VEvent(/* initialise = */ true)
+
         for (processor in fieldProcessors)
             processor.process(from = entity, main = main, to = vEvent)
         return vEvent
     }
 
+    /**
+     * Makes sure that the event has a UID ([Events.UID_2445] in the main event row).
+     *
+     * If the event doesn't have a UID, a new one is generated using [generateUid] and
+     * put into [main].
+     *
+     * @param main  event to be checked (**will be modified** if it doesn't already have a UID)
+     */
+    private fun provideUid(
+        main: Entity,
+        generateUid: () -> String
+    ): String {
+        val mainValues = main.entityValues
+        val existingUid = mainValues.getAsString(Events.UID_2445)
+        if (existingUid != null) {
+            // UID already present, nothing to do
+            return existingUid
+        }
+
+        // have a look at extended properties (Google Calendar)
+        val googleCalendarUid = main.subValues.firstOrNull {
+            it.uri == ExtendedProperties.CONTENT_URI &&
+            it.values.getAsString(ExtendedProperties.NAME) == EventsContract.EXTNAME_GOOGLE_CALENDAR_UID
+        }?.values?.getAsString(ExtendedProperties.VALUE)
+        if (googleCalendarUid != null) {
+            // copy to UID_2445 so that it will be processed by UidProcessor and return
+            mainValues.put(Events.UID_2445, googleCalendarUid)
+            return googleCalendarUid
+        }
+
+        // still no UID, generate one
+        val newUid = generateUid()
+        mainValues.put(Events.UID_2445, newUid)
+        return newUid
+    }
+
 
     companion object {
 

lib/src/main/kotlin/at/bitfire/synctools/mapping/calendar/ProdIdGenerator.kt

@@ -0,0 +1,36 @@
+/*
+ * This file is part of bitfireAT/synctools which is released under GPLv3.
+ * Copyright © All Contributors. See the LICENSE and AUTHOR files in the root directory for details.
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+package at.bitfire.synctools.mapping.calendar
+
+fun interface ProdIdGenerator {
+
+    /**
+     * Generates a `PRODID` string using additional package names.
+     *
+     * @param packages  package names that have modified/generated the iCalendar (like `com.example.app.calendar`; may be empty)
+     *
+     * @return the full `PRODID` string, with the package names and probably additional information added
+     * (like `MyApp/1.0 (com.example.app.calendar)`)
+     */
+    fun generateProdId(packages: List<String>): String
+
+}
+
+class DefaultProdIdGenerator(
+    private val baseId: String
+): ProdIdGenerator {
+
+    override fun generateProdId(packages: List<String>): String {
+        val builder = StringBuilder(baseId)
+        if (packages.isNotEmpty())
+            builder .append(" (")
+                .append(packages.joinToString(", "))
+                .append(")")
+        return builder.toString()
+    }
+
+}

lib/src/main/kotlin/at/bitfire/synctools/mapping/calendar/builder/OrganizerBuilder.kt

@@ -13,10 +13,13 @@ import net.fortuna.ical4j.model.Property
 import net.fortuna.ical4j.model.component.VEvent
 import net.fortuna.ical4j.model.parameter.Email
 import net.fortuna.ical4j.model.property.Attendee
+import net.fortuna.ical4j.model.property.Organizer
+import java.net.URI
+import java.util.logging.Level
 import java.util.logging.Logger
 
 class OrganizerBuilder(
-    private val ownerAccount: String?
+    private val ownerAccount: String
 ): AndroidEntityBuilder {
 
     private val logger
@@ -27,24 +30,33 @@ class OrganizerBuilder(
         val groupScheduled = from.getProperties<Attendee>(Property.ATTENDEE).isNotEmpty()
         if (groupScheduled) {
             values.put(Events.HAS_ATTENDEE_DATA, 1)
-            values.put(Events.ORGANIZER, from.organizer?.let { organizer ->
-                val uri = organizer.calAddress
-                val email = if (uri.scheme.equals("mailto", true))
-                    uri.schemeSpecificPart
-                else
-                    organizer.getParameter<Email>(Parameter.EMAIL)?.value
 
-                if (email != null)
-                    return@let email
-
-                logger.warning("Ignoring ORGANIZER without email address (not supported by Android)")
-                null
-            } ?: ownerAccount)
+            // We prefer the ORGANIZER from the main event and not the exception (it must be the same).
+            // See RFC 6638 3.1 and 3.2.4.2.
+            values.put(Events.ORGANIZER, emailFromOrganizer(main.organizer ?: from.organizer) ?: ownerAccount)
 
         } else { /* !groupScheduled */
             values.put(Events.HAS_ATTENDEE_DATA, 0)
             values.put(Events.ORGANIZER, ownerAccount)
         }
     }
 
+    fun emailFromOrganizer(organizer: Organizer?): String? {
+        if (organizer == null)
+            return null
+
+        // Take from mailto: value or EMAIL parameter
+        val uri: URI? = organizer.calAddress
+        val email = if (uri?.scheme.equals("mailto", true))
+            uri?.schemeSpecificPart
+        else
+            organizer.getParameter<Email>(Parameter.EMAIL)?.value
+
+        if (email != null)
+            return email
+
+        logger.log(Level.WARNING, "Ignoring ORGANIZER without email address (not supported by Android)", organizer)
+        return null
+    }
+
 }

lib/src/main/kotlin/at/bitfire/synctools/mapping/calendar/builder/UidBuilder.kt

@@ -13,7 +13,9 @@ import net.fortuna.ical4j.model.component.VEvent
 class UidBuilder: AndroidEntityBuilder {
 
     override fun build(from: VEvent, main: VEvent, to: Entity) {
-        to.entityValues.put(Events.UID_2445, from.uid?.value)
+        // Always take UID from main event because exceptions must have the same UID anyway.
+        // Note: RFC 5545 requires UID for VEVENTs, however the obsoleted RFC 2445 does not.
+        to.entityValues.put(Events.UID_2445, main.uid?.value)
     }
 
 }