diff --git a/build.gradle b/build.gradle index 85e15b083..5ecf154da 100644 --- a/build.gradle +++ b/build.gradle @@ -27,6 +27,7 @@ buildscript { classpath "org.jetbrains.dokka:dokka-android-gradle-plugin:$dokka_android_gradle_plugin_version" classpath("org.jetbrains.dokka:dokka-gradle-plugin:$dokka_version") classpath "org.jetbrains.kotlin:kotlin-serialization:$kotlin_version" + classpath "androidx.room:room-gradle-plugin:$room_version" } } @@ -58,6 +59,7 @@ subprojects { apply plugin: 'com.android.library' apply plugin: 'kotlin-android' apply plugin: 'idea' + apply plugin: 'kotlin-kapt' //---------------------------------------------------------------------------// // Sources and classpath configurations // diff --git a/gradle.properties b/gradle.properties index 8e434a535..75682ce72 100644 --- a/gradle.properties +++ b/gradle.properties @@ -36,6 +36,8 @@ dokka_android_gradle_plugin_version=0.9.18 dokka_version=1.9.20 publish_plugin_version=2.0.0 versions_plugin_version=0.51.0 +room_version=2.6.1 +paging_version=3.3.6 radar_commons_version=1.1.2 radar_schemas_commons_version=0.8.11 diff --git a/plugins/radar-android-application-status/README.md b/plugins/radar-android-application-status/README.md index e4e28c49c..6f6d1b684 100644 --- a/plugins/radar-android-application-status/README.md +++ b/plugins/radar-android-application-status/README.md @@ -22,20 +22,27 @@ To activate this plugin, add the provider `application_status` to the Firebase R This plugin takes the following Firebase configuration parameters: -| Name | Type | Default | Description | -| ---- | ---- | ------- | ----------- | -| `ntp_server` | string | `` | NTP server to synchronize time with. If empty, time is not synchronized and the `application_external_time` topic will not receive data. | -| `application_status_update_rate` | int (seconds) | `300` = 5 minutes | Rate at which to send data for all application topics. | -| `application_send_ip` | boolean | `false` | Whether to send the device IP address with the server status. | -| `application_time_zone_update_rate` | int (seconds) | `86400` = 1 day | How often to send the current time zone. Set to `0` to disable. | +| Name | Type | Default | Description | +|------------------------------------------------|---------------|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ntp_server` | string | `` | NTP server to synchronize time with. If empty, time is not synchronized and the `application_external_time` topic will not receive data. | +| `application_status_update_rate` | int (seconds) | `300` = 5 minutes | Rate at which to send data for all application topics. | +| `application_send_ip` | boolean | `false` | Whether to send the device IP address with the server status. | +| `application_time_zone_update_rate` | int (seconds) | `86400` = 1 day | How often to send the current time zone. Set to `0` to disable. | +| `application_metrics_verification_update_rate` | int (seconds) | `300` = 5 minutes | Defines the interval for verifying that records older than application_metrics_retention_time are not present. Set to 0 to disable. | +| `application_metrics_buffer_size` | int (count) | `100` | Specifies the number of records to keep in the buffer before adding them to the database. Storing records in batches significantly improves performance. | +| `application_metrics_retention_time` | int (seconds) | `7*86400` = 7 day | Determines how long application metrics are retained in the database. Records older than this value are automatically deleted. | +| `application_metrics_data_retention_count` | int (seconds) | `10000` = 10000 records | Specifies the maximum number of messages to retain for application metrics in the database. When this limit is exceeded, older messages are deleted. | This plugin produces data for the following topics: (types starts with `org.radarcns.monitor.application` prefix) -| Topic | Type | Description | -| ----- | ---- | ----------- | -| `application_external_time` | `ApplicationExternalTime` | External NTP time. Requires `ntp_server` parameter to be set. | -| `application_record_counts` | `ApplicationRecordCounts` | Number of records sent and in queue. | -| `application_uptime` | `ApplicationUptime` | Time since the device booted. | -| `application_server_status` | `ApplicationServerStatus` | Server connection status. | -| `application_time_zone` | `ApplicationTimeZone` | Application time zone. Data is only sent on updates. | -| `application_device_info` | `ApplicationDeviceInfo` | Device information. Data is only sent on updates. | +| Topic | Type | Description | +|----------------------------------|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `application_external_time` | `ApplicationExternalTime` | External NTP time. Requires `ntp_server` parameter to be set. | +| `application_record_counts` | `ApplicationRecordCounts` | Number of records sent and in queue. | +| `application_uptime` | `ApplicationUptime` | Time since the device booted. | +| `application_server_status` | `ApplicationServerStatus` | Server connection status. | +| `application_time_zone` | `ApplicationTimeZone` | Application time zone. Data is only sent on updates. | +| `application_device_info` | `ApplicationDeviceInfo` | Device information. Data is only sent on updates. | +| `application_network_status` | `ApplicationNetworkStatus` | Represents the network connectivity status of an application, indicating whether it is connected to the internet and, if so, whether the connection is via Wi-Fi or cellular data. | +| `application_plugin_status` | `ApplicationPluginStatus` | Represents the status of a plugin, indicating its current connection state. | +| `application_topic_records_sent` | `ApplicationTopicRecordsSent` | Number of records sent for the topic since the last upload | diff --git a/plugins/radar-android-application-status/build.gradle b/plugins/radar-android-application-status/build.gradle index ba5b67c10..0c30d95bd 100644 --- a/plugins/radar-android-application-status/build.gradle +++ b/plugins/radar-android-application-status/build.gradle @@ -13,9 +13,31 @@ description = "Application statistics plugin for RADAR passive remote monitoring // Sources and classpath configurations // //---------------------------------------------------------------------------// +android { + buildFeatures { + viewBinding true + } +} + dependencies { api project(":radar-commons-android") implementation "androidx.localbroadcastmanager:localbroadcastmanager:$localbroadcastmanager_version" + implementation "androidx.appcompat:appcompat:$appcompat_version" + implementation "com.google.android.material:material:$material_version" + implementation "androidx.activity:activity:1.9.0" + implementation "androidx.constraintlayout:constraintlayout:$constraintlayout_version" + implementation "androidx.legacy:legacy-support-v4:$legacy_support_version" + implementation 'androidx.fragment:fragment-ktx:1.8.1' + + implementation "androidx.room:room-runtime:$room_version" + kapt "androidx.room:room-compiler:$room_version" + implementation "androidx.room:room-ktx:$room_version" // For Kotlin Coroutines support + +} + +afterEvaluate { + tasks["dokkaJavadoc"].dependsOn(tasks.getByName("kaptReleaseKotlin"), tasks.getByName("kaptDebugKotlin")) } apply from: "$rootDir/gradle/publishing.gradle" +apply plugin: 'org.jetbrains.kotlin.android' diff --git a/plugins/radar-android-application-status/src/main/AndroidManifest.xml b/plugins/radar-android-application-status/src/main/AndroidManifest.xml index 635aabfd0..418000ad6 100644 --- a/plugins/radar-android-application-status/src/main/AndroidManifest.xml +++ b/plugins/radar-android-application-status/src/main/AndroidManifest.xml @@ -2,8 +2,14 @@ - + + + - + + \ No newline at end of file diff --git a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationState.kt b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationState.kt index 2219fc9fb..500d5eb57 100755 --- a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationState.kt +++ b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationState.kt @@ -18,9 +18,18 @@ package org.radarbase.monitor.application import org.radarbase.android.kafka.ServerStatus import org.radarbase.android.source.BaseSourceState +import org.radarbase.android.storage.entity.NetworkStatusLog +import org.radarbase.android.storage.entity.SourceStatusLog import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger class ApplicationState : BaseSourceState() { + + var uiManager: UserInterfaceManager? = null + + val sourceStatusBufferCount: AtomicInteger = AtomicInteger(0) + val networkStatusBufferCount: AtomicInteger = AtomicInteger(0) + @set:Synchronized var serverStatus: ServerStatus? = null @Synchronized get() = field ?: ServerStatus.DISCONNECTED @@ -29,10 +38,43 @@ class ApplicationState : BaseSourceState() { var recordsSent = 0L private set + val metricsCountPerTopic: MutableMap = ConcurrentHashMap() val cachedRecords: MutableMap = ConcurrentHashMap() + val recordsSentPerTopic: MutableMap = ConcurrentHashMap() + private val sourceStatusBuffer: MutableList = mutableListOf() + private val networkStatusBuffer: MutableList = mutableListOf() + + fun addSourceStatus(status: SourceStatusLog) { + sourceStatusBuffer.add(status) + } + + fun clearSourceStatuses() { + sourceStatusBuffer.clear() + } + + fun clearNetworkStatuses() { + networkStatusBuffer.clear() + } + + fun getSourceStatuses(): List { + return sourceStatusBuffer + } + + fun addNetworkStatus(status: NetworkStatusLog) { + networkStatusBuffer.add(status) + } + + fun getNetworkStatuses(): List { + return networkStatusBuffer + } @Synchronized fun addRecordsSent(nRecords: Long) { recordsSent += nRecords } + + interface UserInterfaceManager { + fun sendSourceStatus() + fun sendNetworkStatus() + } } diff --git a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusManager.kt b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusManager.kt index a3532170c..576bd7a07 100755 --- a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusManager.kt +++ b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusManager.kt @@ -26,6 +26,7 @@ import kotlinx.coroutines.Deferred import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.async +import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -35,21 +36,32 @@ import org.radarbase.android.data.TableDataHandler import org.radarbase.android.kafka.TopicSendReceipt import org.radarbase.android.source.AbstractSourceManager import org.radarbase.android.source.SourceStatusListener +import org.radarbase.android.source.SourceStatusTrace +import org.radarbase.android.storage.db.RadarApplicationDatabase +import org.radarbase.android.storage.entity.ConnectionStatus +import org.radarbase.android.storage.entity.NetworkStatusLog +import org.radarbase.android.storage.entity.SourceStatusLog import org.radarbase.android.util.ChangeRunner import org.radarbase.android.util.CoroutineTaskExecutor +import org.radarbase.android.util.NetworkConnectedReceiver import org.radarbase.android.util.OfflineProcessor import org.radarbase.android.util.takeTrimmedIfNotEmpty import org.radarbase.monitor.application.ApplicationStatusService.Companion.UPDATE_RATE_DEFAULT +import org.radarbase.monitor.application.utils.AppRepository import org.radarcns.kafka.ObservationKey import org.radarcns.monitor.application.ApplicationDeviceInfo import org.radarcns.monitor.application.ApplicationExternalTime +import org.radarcns.monitor.application.ApplicationNetworkStatus +import org.radarcns.monitor.application.ApplicationPluginStatus import org.radarcns.monitor.application.ApplicationRecordCounts import org.radarcns.monitor.application.ApplicationServerStatus import org.radarcns.monitor.application.ApplicationTimeZone +import org.radarcns.monitor.application.ApplicationTopicRecordsSent import org.radarcns.monitor.application.ApplicationUptime import org.radarcns.monitor.application.ExternalTimeProtocol import org.radarcns.monitor.application.OperatingSystem import org.radarcns.monitor.application.ServerStatus +import org.radarcns.monitor.application.SourceStatus import org.slf4j.LoggerFactory import java.net.InetAddress import java.net.NetworkInterface @@ -60,7 +72,7 @@ import java.util.concurrent.TimeUnit.SECONDS class ApplicationStatusManager( service: ApplicationStatusService -) : AbstractSourceManager(service) { +) : AbstractSourceManager(service), ApplicationState.UserInterfaceManager { private val serverTopic: Deferred> = service.lifecycleScope.async(Dispatchers.Default) { createCache("application_server_status", ApplicationServerStatus()) } @@ -79,15 +91,34 @@ class ApplicationStatusManager( private val deviceInfoTopic: Deferred> = service.lifecycleScope.async(Dispatchers.Default) { createCache("application_device_info", ApplicationDeviceInfo()) } + private val networkStatusTopic: Deferred> = service.lifecycleScope.async(Dispatchers.Default) { + createCache("application_network_status", ApplicationNetworkStatus()) + } + private val pluginStatusTopic: Deferred> = service.lifecycleScope.async(Dispatchers.Default) { + createCache("application_plugin_status", ApplicationPluginStatus()) + } + private val recordCountsPerTopic: Deferred> = service.lifecycleScope.async(Dispatchers.Default) { + createCache("application_topic_records_sent", ApplicationTopicRecordsSent()) + } private val processor: OfflineProcessor private val creationTimeStamp: Long = SystemClock.elapsedRealtime() private val sntpClient: SntpClient = SntpClient() private val prefs: SharedPreferences = service.getSharedPreferences(ApplicationStatusManager::class.java.name, Context.MODE_PRIVATE) private var tzProcessor: OfflineProcessor? = null + private var verificationProcessor: OfflineProcessor? = null private val tzIntervalMutex: Mutex = Mutex() + private val verificationIntervalMutex: Mutex = Mutex() + private val sourceStatusMutex: Mutex = Mutex() + private val networkStatusMutex: Mutex = Mutex() private val applicationStatusExecutor: CoroutineTaskExecutor = CoroutineTaskExecutor(this::class.simpleName!!) + private val radarApplicationDb: RadarApplicationDatabase = RadarApplicationDatabase.getInstance(service) + private val appRepository = AppRepository(radarApplicationDb) + private val sourceStatusDao = appRepository.sourceStatusDao + private val networkStatusDao = appRepository.networkStatusDao + + private val networkStatusReceiver: NetworkConnectedReceiver = NetworkConnectedReceiver(service) @get:Synchronized @set:Synchronized @@ -100,6 +131,10 @@ class ApplicationStatusManager( field = value?.takeTrimmedIfNotEmpty() } + var metricsBatchSize: Long = 0 + var metricsRetentionTime: Long = 0 + var metricsRetentionSize: Long = 0 + private var previousInetAddress: InetAddress? = null private lateinit var tzOffsetCache: ChangeRunner @@ -107,6 +142,11 @@ class ApplicationStatusManager( private var cachedRecordTrackJob: Job? = null private var serverRecordsSentJob: Job? = null private var serverStatusJob: Job? = null + private var sourceStatusJob: Job? = null + private var networkReceiverJob: Job? = null + private var recordsSentPerTopicJob: Job? = null + + private var networkMonitorJob: Job? = null init { name = service.getString(R.string.applicationServiceDisplayName) @@ -117,6 +157,7 @@ class ApplicationStatusManager( ::processRecordsSent, ::processReferenceTime, ::processDeviceInfo, + ::processRecordCountsPerTopic ) requestCode = APPLICATION_PROCESSOR_REQUEST_CODE requestName = APPLICATION_PROCESSOR_REQUEST_NAME @@ -162,36 +203,163 @@ class ApplicationStatusManager( this.tzOffsetCache = ChangeRunner(this.prefs.getInt("timeZoneOffset", -1)) } tzProcessor?.start() + verificationProcessor?.start() + + networkMonitorJob = service.lifecycleScope.launch(Dispatchers.Default) { + networkStatusReceiver.monitor() + } logger.info("Starting ApplicationStatusManager") - with(applicationStatusExecutor) { - service.dataHandler?.let { handler -> - cachedRecordTrackJob = service.lifecycleScope.launch(Dispatchers.Default) { - handler.numberOfRecords - .collect { records: TableDataHandler.CacheSize -> - val topic = records.topicName - val noOfRecords = records.numberOfRecords - logger.trace("Topic {} has {} records in cache", topic, noOfRecords) - state.cachedRecords[topic] = noOfRecords.coerceAtLeast(0) + state.uiManager = this@ApplicationStatusManager + + service.dataHandler?.let { handler -> + cachedRecordTrackJob = service.lifecycleScope.launch(Dispatchers.Default) { + handler.numberOfRecords + .collect { records: TableDataHandler.CacheSize -> + val topic = records.topicName + val noOfRecords = records.numberOfRecords + logger.trace("Topic {} has {} records in cache", topic, noOfRecords) + state.cachedRecords[topic] = noOfRecords.coerceAtLeast(0) + } + } + + serverRecordsSentJob = service.lifecycleScope.launch(Dispatchers.Default) { + handler.recordsSent.collect { sent: TopicSendReceipt -> + logger.trace("Topic {} sent {} records", sent.topic, sent.numberOfRecords) + state.addRecordsSent(sent.numberOfRecords) + } + } + + serverStatusJob = service.lifecycleScope.launch(Dispatchers.Default) { + handler.serverStatus.collect { + state.serverStatus = it + logger.trace("Updated Server Status to {}", it) + } + } + + sourceStatusJob = service.lifecycleScope.launch(Dispatchers.Default) { + var shouldClearCountOnNextBatchUpdate = false + + handler.sourceStatus + .collect { trace: SourceStatusTrace -> + sourceStatusMutex.withLock { + logger.debug( + "Received source status: {} for plugin {}", + trace.status, + trace.plugin + ) + + val pluginName = trace.plugin + pluginName?.let { + var bufferCount = state.sourceStatusBufferCount.get() + if (bufferCount >= metricsBatchSize) { + launch(Dispatchers.IO) { + insertSourceStatusBatchToDb(shouldClearCountOnNextBatchUpdate) + shouldClearCountOnNextBatchUpdate = false + } + } + bufferCount = state.sourceStatusBufferCount.get() + + if (bufferCount == 0) { + withContext(Dispatchers.Default) { + state.clearSourceStatuses() + } + } + + val totalSourceStatusMetrics = state.metricsCountPerTopic.compute(APPLICATION_SOURCE_STATUS_TOPIC) { _, count -> + (count ?: 0) + 1 + } ?: 0 + + if (!shouldClearCountOnNextBatchUpdate && totalSourceStatusMetrics > metricsRetentionSize) { + shouldClearCountOnNextBatchUpdate = true + } + + state.addSourceStatus( + SourceStatusLog( + time = currentTime.toLong(), + plugin = it, + sourceStatus = trace.status + ) + ) + + state.sourceStatusBufferCount.incrementAndGet() } + ?: logger.debug("Plugin name is null when processing status") } + } + } + + networkReceiverJob = service.lifecycleScope.launch(Dispatchers.Default) { + var shouldClearCountOnNextBatchUpdate = false + networkStatusReceiver.state!!. + distinctUntilChanged(). + collect { status: NetworkConnectedReceiver.NetworkState -> + networkStatusMutex.withLock { + logger.debug("Received network state: {}", status) + + val networkStatusBufferCount = state.networkStatusBufferCount.get() + + logger.debug( + "Network status buffer count: {}", + networkStatusBufferCount + ) - serverRecordsSentJob = service.lifecycleScope.launch(Dispatchers.Default) { - handler.recordsSent.collect { sent: TopicSendReceipt -> - logger.trace("Topic {} sent {} records", sent.topic, sent.numberOfRecords) - state.addRecordsSent(sent.numberOfRecords) + if (networkStatusBufferCount == 0) { + launch(Dispatchers.Default) { + state.clearNetworkStatuses() + } + } + if (networkStatusBufferCount > metricsBatchSize) { + launch(Dispatchers.IO) { + insertNetworkStatusBatchToDb(shouldClearCountOnNextBatchUpdate) + shouldClearCountOnNextBatchUpdate = false } } - serverStatusJob = service.lifecycleScope.launch(Dispatchers.Default) { - handler.serverStatus.collect { - state.serverStatus = it - logger.trace("Updated Server Status to {}", it) + state.metricsCountPerTopic.compute(APPLICATION_NETWORK_STATUS_TOPIC) { _, count -> + (count ?: 0) + 1 } + val totalNetworkStatusMetrics = + state.metricsCountPerTopic[APPLICATION_NETWORK_STATUS_TOPIC] ?: 0 + if (!shouldClearCountOnNextBatchUpdate && totalNetworkStatusMetrics > metricsRetentionSize) { + shouldClearCountOnNextBatchUpdate = true + } + + val connectionStatus: ConnectionStatus = when { + status is NetworkConnectedReceiver.NetworkState.Disconnected -> ConnectionStatus.DISCONNECTED + status is NetworkConnectedReceiver.NetworkState.Connected && status.hasConnection( + true + ) -> ConnectionStatus.CONNECTED_WIFI + + else -> ConnectionStatus.CONNECTED_CELLULAR + } + + state.addNetworkStatus( + NetworkStatusLog( + time = currentTime.toLong(), + connectionStatus = connectionStatus + ) + ) + + state.networkStatusBufferCount.incrementAndGet() } } } + recordsSentPerTopicJob = service.lifecycleScope.launch(Dispatchers.Default) { + handler.recordsSent + .collect { records: TopicSendReceipt -> + logger.debug("{} Records sent for topic: {}", records.numberOfRecords, records.topic) + val topic = records.topic + val noOfRecords = records.numberOfRecords.coerceAtLeast(0) + + state.recordsSentPerTopic.compute(topic) { _, existingRecords -> + (existingRecords ?: 0) + noOfRecords + } + } + } + } + status = SourceStatusListener.Status.CONNECTED } @@ -266,6 +434,103 @@ class ApplicationStatusManager( } } + private suspend fun insertSourceStatusBatchToDb(manageSpace: Boolean) { + withContext(Dispatchers.Default) { + state.sourceStatusBufferCount.set(0) + } + + logger.info("Adding source status batch to database") + val statuses = ArrayList(state.getSourceStatuses()) + sourceStatusDao.addAll(statuses) + if (manageSpace) { + val numbersDeleted = sourceStatusDao.deleteStatusesCountGreaterThan(metricsRetentionSize) + state.metricsCountPerTopic[APPLICATION_SOURCE_STATUS_TOPIC] ?: return + state.metricsCountPerTopic.compute(APPLICATION_SOURCE_STATUS_TOPIC) { _, count -> + count!! - numbersDeleted + } + } + } + + private suspend fun insertNetworkStatusBatchToDb(manageSpace: Boolean) { + withContext(Dispatchers.Default) { + state.networkStatusBufferCount.set(0) + } + + logger.info("Adding network status batch to database") + val networkStatuses = ArrayList(state.getNetworkStatuses()) + networkStatusDao.addAll(networkStatuses) + if (manageSpace) { + logger.debug("Managing space") + val deletedCount = networkStatusDao.deleteNetworkLogsCountGreaterThan(metricsRetentionSize) + state.metricsCountPerTopic[APPLICATION_NETWORK_STATUS_TOPIC] ?: return + state.metricsCountPerTopic.compute(APPLICATION_NETWORK_STATUS_TOPIC) { _, count -> + count!! - deletedCount + } + } + } + + override fun sendSourceStatus() { + logger.debug("Sending Source Status") + service.lifecycleScope.launch { + var maxTime = 0L + + val sourceStatusLogs = withContext(Dispatchers.IO) { + sourceStatusDao.loadAllStatuses() + } + + sourceStatusLogs.forEach { status -> + if (maxTime < status.time) { + maxTime = status.time + } + + val state: SourceStatus = status.sourceStatus.toPluginState() + send(pluginStatusTopic.await(), ApplicationPluginStatus(status.time.toDouble(), status.plugin, state)) + } + + launch(Dispatchers.IO) { + val statusDeletedCount = sourceStatusDao.deleteSourceLogsOlderThan(maxTime) + + val sourceStatusCount = state.metricsCountPerTopic[APPLICATION_SOURCE_STATUS_TOPIC] + + if (sourceStatusCount != null) { + state.metricsCountPerTopic[APPLICATION_SOURCE_STATUS_TOPIC] = + sourceStatusCount - statusDeletedCount + logger.debug("All source logs sent") + } + } + } + } + + override fun sendNetworkStatus() { + logger.debug("Sending Network Status") + service.lifecycleScope.launch { + var latestTime = 0L + + val networkStatusLogs = withContext(Dispatchers.IO) { + networkStatusDao.loadAllNetworkLogs() + } + + networkStatusLogs.forEach { status -> + if (latestTime < status.time) { + latestTime = status.time + } + val state = status.connectionStatus.toConnectionState() + send(networkStatusTopic.await(), ApplicationNetworkStatus(status.time.toDouble(), state)) + } + + launch(Dispatchers.IO) { + val noOfLogsDeleted = networkStatusDao.deleteNetworkLogsOlderThan(latestTime) + val count = state.metricsCountPerTopic[APPLICATION_NETWORK_STATUS_TOPIC] + + if (count != null) { + state.metricsCountPerTopic[APPLICATION_NETWORK_STATUS_TOPIC] = + count - noOfLogsDeleted + logger.debug("All network logs sent") + } + } + } + } + private suspend fun processServerStatus() { val time = currentTime @@ -292,6 +557,26 @@ class ApplicationStatusManager( return previousInetAddress?.hostAddress } + private suspend fun processRecordCountsPerTopic() { + state.recordsSentPerTopic.forEach { + send(recordCountsPerTopic.await(), ApplicationTopicRecordsSent(currentTime, it.key, it.value)) + } + state.recordsSentPerTopic.clear() + } + + private suspend fun processVerifyRecordsRetention() { + val currentTime = currentTime + val retentionThreshold: Long = (currentTime - metricsRetentionTime).toLong() + service.lifecycleScope.launch(Dispatchers.IO) { + launch { + sourceStatusDao.deleteSourceLogsOlderThan(retentionThreshold) + } + launch { + networkStatusDao.deleteNetworkLogsOlderThan(retentionThreshold) + } + } + } + private suspend fun processUptime() { val uptime = (SystemClock.elapsedRealtime() - creationTimeStamp) / 1000.0 send(uptimeTopic.await(), ApplicationUptime(currentTime, uptime)) @@ -314,6 +599,10 @@ class ApplicationStatusManager( override fun onClose() { applicationStatusExecutor.stop { this.processor.stop() + networkMonitorJob?.also { + it.cancel() + networkMonitorJob = null + } cachedRecordTrackJob?.also { it.cancel() cachedRecordTrackJob = null @@ -326,7 +615,20 @@ class ApplicationStatusManager( it.cancel() serverStatusJob = null } + sourceStatusJob?.also { + it.cancel() + sourceStatusJob = null + } + recordsSentPerTopicJob?.also { + it.cancel() + recordsSentPerTopicJob = null + } + networkReceiverJob?.also { + it.cancel() + networkReceiverJob = null + } tzProcessor?.stop() + verificationProcessor?.stop() } } @@ -384,6 +686,39 @@ class ApplicationStatusManager( } } + fun setVerificationUpdateRate(verificationUpdateRate: Long, unit: TimeUnit) { + service.lifecycleScope.launch(Dispatchers.Default) { + verificationIntervalMutex.withLock { + if (verificationUpdateRate > 0) { + var processor = this@ApplicationStatusManager.verificationProcessor + if (processor == null) { + processor = OfflineProcessor(service) { + process = listOf { this@ApplicationStatusManager.processVerifyRecordsRetention() } + requestCode = APPLICATION_VERIFICATION_PROCESSOR_REQUEST_CODE + requestName = APPLICATION_VERIFICATION_PROCESSOR_REQUEST_NAME + interval(verificationUpdateRate, unit) + wake = false + } + this@ApplicationStatusManager.verificationProcessor = processor + + if (this@ApplicationStatusManager.state.status == SourceStatusListener.Status.CONNECTED) { + processor.start() + } + } else { + processor.interval(verificationUpdateRate, unit) + } + } else { + this@ApplicationStatusManager.verificationProcessor?.let { + service.lifecycleScope.launch(Dispatchers.Default) { + it.stop() + } + this@ApplicationStatusManager.verificationProcessor = null + } + } + } + } + } + fun setApplicationStatusUpdateRate(period: Long, unit: TimeUnit) { processor.interval(period, unit) } @@ -402,11 +737,32 @@ class ApplicationStatusManager( private const val NUMBER_UNKNOWN = -1L private const val APPLICATION_PROCESSOR_REQUEST_CODE = 72553575 private const val APPLICATION_TZ_PROCESSOR_REQUEST_CODE = 72553576 + private const val APPLICATION_VERIFICATION_PROCESSOR_REQUEST_CODE = 72553577 private const val APPLICATION_PROCESSOR_REQUEST_NAME = "org.radarbase.monitor.application.ApplicationStatusManager" private const val APPLICATION_TZ_PROCESSOR_REQUEST_NAME = "$APPLICATION_PROCESSOR_REQUEST_NAME.timeZone" + private const val APPLICATION_VERIFICATION_PROCESSOR_REQUEST_NAME = "$APPLICATION_PROCESSOR_REQUEST_NAME.verification" + private const val APPLICATION_SOURCE_STATUS_TOPIC = "application_plugin_status" + private const val APPLICATION_NETWORK_STATUS_TOPIC = "application_network_status" private fun Long.toIntCapped(): Int = if (this <= Int.MAX_VALUE) toInt() else Int.MAX_VALUE + private fun SourceStatusListener.Status.toPluginState(): SourceStatus = when(this) { + SourceStatusListener.Status.READY -> SourceStatus.READY + SourceStatusListener.Status.CONNECTING -> SourceStatus.CONNECTING + SourceStatusListener.Status.CONNECTED -> SourceStatus.CONNECTED + SourceStatusListener.Status.DISCONNECTING -> SourceStatus.DISCONNECTING + SourceStatusListener.Status.DISCONNECTED -> SourceStatus.DISCONNECTED + SourceStatusListener.Status.UNAVAILABLE -> SourceStatus.UNAVAILABLE + else -> SourceStatus.UNKNOWN + } + + private fun ConnectionStatus.toConnectionState(): org.radarcns.monitor.application.ConnectionStatus = when(this) { + ConnectionStatus.CONNECTED_WIFI -> org.radarcns.monitor.application.ConnectionStatus.CONNECTED_WIFI + ConnectionStatus.CONNECTED_CELLULAR -> org.radarcns.monitor.application.ConnectionStatus.CONNECTED_CELLULAR + ConnectionStatus.DISCONNECTED -> org.radarcns.monitor.application.ConnectionStatus.DISCONNECTED + else -> org.radarcns.monitor.application.ConnectionStatus.UNKNOWN + } + private fun org.radarbase.android.kafka.ServerStatus?.toServerStatus(): ServerStatus = when (this) { org.radarbase.android.kafka.ServerStatus.CONNECTED, org.radarbase.android.kafka.ServerStatus.READY, diff --git a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusProvider.kt b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusProvider.kt index 4febe454d..85040212b 100644 --- a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusProvider.kt +++ b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusProvider.kt @@ -16,9 +16,11 @@ package org.radarbase.monitor.application +import android.content.Intent import org.radarbase.android.BuildConfig import org.radarbase.android.RadarService import org.radarbase.android.source.SourceProvider +import org.radarbase.monitor.application.ui.ApplicationMetricsActivity open class ApplicationStatusProvider(radarService: RadarService) : SourceProvider(radarService) { override val description: String? @@ -46,4 +48,9 @@ open class ApplicationStatusProvider(radarService: RadarService) : SourceProvide override val sourceModel: String = "pRMT" override val version: String = BuildConfig.VERSION_NAME + + override val actions: List + get() = listOf(Action(radarService.getString(R.string.start_app_metrics_activity)){ + startActivity(Intent(this, ApplicationMetricsActivity::class.java)) + }) } diff --git a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusService.kt b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusService.kt index 134bcd724..0616dd758 100755 --- a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusService.kt +++ b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ApplicationStatusService.kt @@ -32,15 +32,27 @@ class ApplicationStatusService : SourceService() { manager as ApplicationStatusManager manager.setApplicationStatusUpdateRate(config.getLong(UPDATE_RATE, UPDATE_RATE_DEFAULT), TimeUnit.SECONDS) manager.setTzUpdateRate(config.getLong(TZ_UPDATE_RATE, TZ_UPDATE_RATE_DEFAULT), TimeUnit.SECONDS) + manager.setVerificationUpdateRate(config.getLong(VERIFICATION_UPDATE_RATE, VERIFICATION_UPDATE_RATE_DEFAULT), TimeUnit.SECONDS) manager.ntpServer = config.optString(NTP_SERVER_CONFIG) manager.isProcessingIp = config.getBoolean(SEND_IP, false) + manager.metricsBatchSize = config.getLong(APPLICATION_METRICS_BATCH_SIZE, APPLICATION_METRICS_BATCH_SIZE_DEFAULT) + manager.metricsRetentionSize = config.getLong(APPLICATION_METRICS_DATA_RETENTION_COUNT, APPLICATION_METRICS_DATA_RETENTION_COUNT_DEFAULT) + manager.metricsRetentionTime = config.getLong(APPLICATION_METRICS_RETENTION_TIME, APPLICATION_METRICS_RETENTION_TIME_DEFAULT) } companion object { private const val UPDATE_RATE = "application_status_update_rate" private const val TZ_UPDATE_RATE = "application_time_zone_update_rate" + private const val VERIFICATION_UPDATE_RATE = "application_metrics_verification_update_rate" + private const val APPLICATION_METRICS_BATCH_SIZE = "application_metrics_buffer_size" + private const val APPLICATION_METRICS_RETENTION_TIME = "application_metrics_retention_time" + private const val APPLICATION_METRICS_DATA_RETENTION_COUNT = "application_metrics_data_retention_count" + private const val APPLICATION_METRICS_BATCH_SIZE_DEFAULT = 5L + private const val APPLICATION_METRICS_RETENTION_TIME_DEFAULT = 7 * 86400L // seconds == 1 day + private const val APPLICATION_METRICS_DATA_RETENTION_COUNT_DEFAULT = 10000L // 10,000 counts per topic private const val SEND_IP = "application_send_ip" internal const val UPDATE_RATE_DEFAULT = 300L // seconds == 5 minutes + internal const val VERIFICATION_UPDATE_RATE_DEFAULT = 3600L // seconds == 1 hour internal const val TZ_UPDATE_RATE_DEFAULT = 86400L // seconds == 1 day private const val NTP_SERVER_CONFIG = "ntp_server" } diff --git a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ui/ApplicationMetricsActivity.kt b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ui/ApplicationMetricsActivity.kt new file mode 100644 index 000000000..421880abb --- /dev/null +++ b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ui/ApplicationMetricsActivity.kt @@ -0,0 +1,131 @@ +package org.radarbase.monitor.application.ui + +import android.content.ComponentName +import android.content.Intent +import android.content.ServiceConnection +import android.os.Bundle +import android.os.IBinder +import android.widget.Toast +import androidx.activity.enableEdgeToEdge +import androidx.activity.viewModels +import androidx.appcompat.app.AppCompatActivity +import androidx.core.view.ViewCompat +import androidx.core.view.WindowInsetsCompat +import androidx.lifecycle.lifecycleScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.radarbase.android.IRadarBinder +import org.radarbase.android.RadarApplication.Companion.radarApp +import org.radarbase.android.storage.db.RadarApplicationDatabase +import org.radarbase.android.util.Boast +import org.radarbase.monitor.application.ApplicationState +import org.radarbase.monitor.application.ApplicationStatusProvider +import org.radarbase.monitor.application.R +import org.radarbase.monitor.application.databinding.ActivityApplicationStatusBinding +import org.radarbase.monitor.application.ui.viewmodel.ApplicationMetricsViewModel +import org.radarbase.monitor.application.ui.viewmodel.factory.ApplicationMetricsViewModelFactory +import org.radarbase.monitor.application.utils.AppRepository +import org.slf4j.LoggerFactory + +class ApplicationMetricsActivity : AppCompatActivity() { + private lateinit var binding: ActivityApplicationStatusBinding + private lateinit var database: RadarApplicationDatabase + private lateinit var repository: AppRepository + private val viewModel: ApplicationMetricsViewModel by viewModels { ApplicationMetricsViewModelFactory(repository) } + + private var statusProvider: ApplicationStatusProvider? = null + private val state: ApplicationState? + get() = statusProvider?.connection?.sourceState + + private val radarServiceConnection = object : ServiceConnection { + override fun onServiceConnected(name: ComponentName?, service: IBinder?) { + logger.debug("Service bound to ApplicationMetricsActivity") + val radarService = service as IRadarBinder + + radarService.connections.forEach { provider -> + if (provider is ApplicationStatusProvider) { + statusProvider = provider + } + } + if (state == null) { + logger.info("Can't send metrics state is null") + Boast.makeText(this@ApplicationMetricsActivity, + R.string.unable_to_proceed_toast, Toast.LENGTH_SHORT).show(true) + return + } + } + + override fun onServiceDisconnected(name: ComponentName?) { + statusProvider = null + } + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + enableEdgeToEdge() + binding = ActivityApplicationStatusBinding.inflate(layoutInflater) + setContentView(binding.root) + ViewCompat.setOnApplyWindowInsetsListener(findViewById(R.id.clAppPlugin)) { v, insets -> + val systemBars = insets.getInsets(WindowInsetsCompat.Type.systemBars()) + v.setPadding(systemBars.left, systemBars.top, systemBars.right, systemBars.bottom) + insets + } + + database = RadarApplicationDatabase.getInstance(applicationContext) + repository = AppRepository(database) + + binding.btnSend.setOnClickListener { + val uiManagerInterface = state?.uiManager ?: run { + logger.warn("Can't send the application metrics, state is null") + return@setOnClickListener + } + + lifecycleScope.launch(Dispatchers.Default) { + val sourceStatusDeferred = async { + uiManagerInterface.sendSourceStatus() + } + val networkStatusDeferred = async { + uiManagerInterface.sendNetworkStatus() + } + sourceStatusDeferred.await() + networkStatusDeferred.await() + withContext(Dispatchers.Main) { + finish() + } + } + } + + updateStatusCounts() + } + + override fun onStart() { + super.onStart() + bindService(Intent(this, radarApp.radarService), radarServiceConnection, 0) + } + + + private fun updateStatusCounts() { + logger.debug("ApplicationStatusDebug: Updating Status count") + + viewModel.loadSourceStatusCount().observe(this@ApplicationMetricsActivity) { sourceCount -> + logger.debug("ApplicationStatusDebug: Source status count: $sourceCount") + binding.tvPluginStatus.text = getString(R.string.plugin_log_count, sourceCount) + } + + viewModel.loadNetworkStatusCount().observe(this@ApplicationMetricsActivity) { networkCount -> + logger.debug("ApplicationStatusDebug: Network status count: $networkCount") + binding.tvNetworkStatus.text = getString(R.string.network_log_count, networkCount) + } + } + + override fun onStop() { + super.onStop() + unbindService(radarServiceConnection) + } + + companion object { + private val logger = LoggerFactory.getLogger(ApplicationMetricsActivity::class.java) + } +} \ No newline at end of file diff --git a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ui/viewmodel/ApplicationMetricsViewModel.kt b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ui/viewmodel/ApplicationMetricsViewModel.kt new file mode 100644 index 000000000..40a9c94ba --- /dev/null +++ b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ui/viewmodel/ApplicationMetricsViewModel.kt @@ -0,0 +1,19 @@ +package org.radarbase.monitor.application.ui.viewmodel + +import androidx.lifecycle.LiveData +import androidx.lifecycle.ViewModel +import org.radarbase.android.storage.dao.NetworkStatusDao +import org.radarbase.android.storage.dao.SourceStatusDao + +class ApplicationMetricsViewModel( + private val sourceStatusDao: SourceStatusDao, + private val networkStatusDao: NetworkStatusDao +) : ViewModel() { + fun loadSourceStatusCount(): LiveData { + return sourceStatusDao.getStatusesCount() + } + + fun loadNetworkStatusCount(): LiveData { + return networkStatusDao.getStatusesCount() + } +} \ No newline at end of file diff --git a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ui/viewmodel/factory/ApplicationMetricsViewModelFactory.kt b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ui/viewmodel/factory/ApplicationMetricsViewModelFactory.kt new file mode 100644 index 000000000..54a720b0b --- /dev/null +++ b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/ui/viewmodel/factory/ApplicationMetricsViewModelFactory.kt @@ -0,0 +1,19 @@ +package org.radarbase.monitor.application.ui.viewmodel.factory + +import androidx.lifecycle.ViewModel +import androidx.lifecycle.ViewModelProvider +import org.radarbase.monitor.application.ui.viewmodel.ApplicationMetricsViewModel +import org.radarbase.monitor.application.utils.AppRepository + +class ApplicationMetricsViewModelFactory(private val repository: AppRepository) : ViewModelProvider.Factory { + override fun create(modelClass: Class): T { + if (modelClass.isAssignableFrom(ApplicationMetricsViewModel::class.java)) { + @Suppress("UNCHECKED_CAST") + return ApplicationMetricsViewModel( + repository.sourceStatusDao, + repository.networkStatusDao + ) as T + } + throw IllegalArgumentException("Unknown ViewModel class") + } +} diff --git a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/utils/AppRepository.kt b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/utils/AppRepository.kt new file mode 100644 index 000000000..e6c0d7364 --- /dev/null +++ b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/utils/AppRepository.kt @@ -0,0 +1,10 @@ +package org.radarbase.monitor.application.utils + +import org.radarbase.android.storage.dao.NetworkStatusDao +import org.radarbase.android.storage.dao.SourceStatusDao +import org.radarbase.android.storage.db.RadarApplicationDatabase + +class AppRepository(db: RadarApplicationDatabase) { + val sourceStatusDao: SourceStatusDao = db.sourceStatusDao() + val networkStatusDao: NetworkStatusDao = db.networkStatusDao() +} diff --git a/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/utils/Utils.kt b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/utils/Utils.kt new file mode 100644 index 000000000..b3dd6e762 --- /dev/null +++ b/plugins/radar-android-application-status/src/main/java/org/radarbase/monitor/application/utils/Utils.kt @@ -0,0 +1,12 @@ +package org.radarbase.monitor.application.utils + +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter + +fun dateTimeFromInstant(epochSeconds: Long): String { + return DateTimeFormatter + .ofPattern("yyyy-MM-dd HH:mm:ss") + .withZone(ZoneId.systemDefault()) + .format(Instant.ofEpochSecond(epochSeconds)) +} \ No newline at end of file diff --git a/plugins/radar-android-application-status/src/main/res/layout/activity_application_status.xml b/plugins/radar-android-application-status/src/main/res/layout/activity_application_status.xml new file mode 100644 index 000000000..a73cf8585 --- /dev/null +++ b/plugins/radar-android-application-status/src/main/res/layout/activity_application_status.xml @@ -0,0 +1,53 @@ + + + +