perf: message logger (#50)

* pref: message logger cache --------- Co-authored-by: rhunk <101876869+rhunk@users.noreply.github.com>
2023-06-09 03:19:40 +02:00
parent 0b80489a1d
commit 20c1f6a779
10 changed files with 212 additions and 97 deletions
--- a/app/src/main/kotlin/me/rhunk/snapenhance/bridge/AbstractBridgeClient.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/bridge/AbstractBridgeClient.kt
@@ -65,6 +65,14 @@ abstract class AbstractBridgeClient {
     */
    abstract fun downloadContent(url: String, path: String): Boolean

+    /**
+     * Get the content of a logged message from the database
+     *
+     * @param conversationId the ID of the conversation
+     * @return the content of the message
+     */
+    abstract fun getLoggedMessageIds(conversationId: String, limit: Int): List<Long>
+
    /**
     * Get the content of a logged message from the database
     *
--- a/app/src/main/kotlin/me/rhunk/snapenhance/bridge/MessageLoggerWrapper.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/bridge/MessageLoggerWrapper.kt
@@ -46,6 +46,16 @@ class MessageLoggerWrapper(
        return Pair(state, message)
    }

+    fun getMessageIds(conversationId: String, limit: Int): List<Long> {
+        val cursor = database.rawQuery("SELECT message_id FROM messages WHERE conversation_id = ? ORDER BY message_id DESC LIMIT ?", arrayOf(conversationId, limit.toString()))
+        val messageIds = mutableListOf<Long>()
+        while (cursor.moveToNext()) {
+            messageIds.add(cursor.getLong(0))
+        }
+        cursor.close()
+        return messageIds
+    }
+
    fun clearMessages() {
        database.execSQL("DELETE FROM messages")
    }
--- a/app/src/main/kotlin/me/rhunk/snapenhance/bridge/client/RootBridgeClient.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/bridge/client/RootBridgeClient.kt
@@ -65,6 +65,10 @@ class RootBridgeClient : AbstractBridgeClient() {
        return true
    }

+    override fun getLoggedMessageIds(conversationId: String, limit: Int): List<Long> {
+        return messageLoggerWrapper.getMessageIds(conversationId, limit)
+    }
+
    override fun getMessageLoggerMessage(conversationId: String, id: Long): ByteArray? {
        val (state, messageData) = messageLoggerWrapper.getMessage(conversationId, id)
        if (state) {
--- a/app/src/main/kotlin/me/rhunk/snapenhance/bridge/client/ServiceBridgeClient.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/bridge/client/ServiceBridgeClient.kt
@@ -13,6 +13,8 @@ import android.os.HandlerThread
 import android.os.IBinder
 import android.os.Message
 import android.os.Messenger
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.suspendCancellableCoroutine
 import me.rhunk.snapenhance.BuildConfig
 import me.rhunk.snapenhance.Logger.xposedLog
 import me.rhunk.snapenhance.bridge.AbstractBridgeClient
@@ -25,11 +27,13 @@ import me.rhunk.snapenhance.bridge.common.impl.file.FileAccessRequest
 import me.rhunk.snapenhance.bridge.common.impl.file.FileAccessResult
 import me.rhunk.snapenhance.bridge.common.impl.locale.LocaleRequest
 import me.rhunk.snapenhance.bridge.common.impl.locale.LocaleResult
+import me.rhunk.snapenhance.bridge.common.impl.messagelogger.MessageLoggerListResult
 import me.rhunk.snapenhance.bridge.common.impl.messagelogger.MessageLoggerRequest
 import me.rhunk.snapenhance.bridge.common.impl.messagelogger.MessageLoggerResult
 import me.rhunk.snapenhance.bridge.service.BridgeService
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.Executors
+import kotlin.coroutines.resume
 import kotlin.reflect.KClass
 import kotlin.system.exitProcess

@@ -59,23 +63,20 @@ class ServiceBridgeClient: AbstractBridgeClient(), ServiceConnection {
    }

    private fun handleResponseMessage(
-        msg: Message,
-        future: CompletableFuture<BridgeMessage>
-    ) {
+        msg: Message
+    ): BridgeMessage {
        val message: BridgeMessage = when (BridgeMessageType.fromValue(msg.what)) {
            BridgeMessageType.FILE_ACCESS_RESULT -> FileAccessResult()
            BridgeMessageType.DOWNLOAD_CONTENT_RESULT -> DownloadContentResult()
            BridgeMessageType.MESSAGE_LOGGER_RESULT -> MessageLoggerResult()
+            BridgeMessageType.MESSAGE_LOGGER_LIST_RESULT -> MessageLoggerListResult()
            BridgeMessageType.LOCALE_RESULT -> LocaleResult()
-            else -> {
-                future.completeExceptionally(IllegalStateException("Unknown message type: ${msg.what}"))
-                return
-            }
+            else -> throw IllegalStateException("Unknown message type: ${msg.what}")
        }

        with(message) {
            read(msg.data)
-            future.complete(this)
+            return this
        }
    }

@@ -84,26 +85,31 @@ class ServiceBridgeClient: AbstractBridgeClient(), ServiceConnection {
        messageType: BridgeMessageType,
        message: BridgeMessage,
        resultType: KClass<T>? = null
-    ): T {
-        val future = CompletableFuture<BridgeMessage>()
+    ) = runBlocking {
+        suspendCancellableCoroutine { cancelableContinuation ->
+            val replyMessenger = Messenger(object : Handler(handlerThread.looper) {
+                override fun handleMessage(msg: Message) {
+                    if (cancelableContinuation.isCancelled) return
+                    runCatching {
+                        cancelableContinuation.resume(handleResponseMessage(msg) as T)
+                    }.onFailure {
+                        cancelableContinuation.cancel(it)
+                    }
+                }
+            })

-        val replyMessenger = Messenger(object : Handler(handlerThread.looper) {
-            override fun handleMessage(msg: Message) {
-                handleResponseMessage(msg, future)
-            }
-        })
-
-        runCatching {
-            with(Message.obtain()) {
-                what = messageType.value
-                replyTo = replyMessenger
-                data = Bundle()
-                message.write(data)
-                messenger.send(this)
+            runCatching {
+                with(Message.obtain()) {
+                    what = messageType.value
+                    replyTo = replyMessenger
+                    data = Bundle()
+                    message.write(data)
+                    messenger.send(this)
+                }
+            }.onFailure {
+                cancelableContinuation.cancel(it)
            }
        }
-
-        return future.get() as T
    }

    override fun createAndReadFile(
@@ -177,6 +183,16 @@ class ServiceBridgeClient: AbstractBridgeClient(), ServiceConnection {
        }
    }

+    override fun getLoggedMessageIds(conversationId: String, limit: Int): List<Long> {
+        sendMessage(
+            BridgeMessageType.MESSAGE_LOGGER_REQUEST,
+            MessageLoggerRequest(MessageLoggerRequest.Action.LIST_IDS, conversationId, limit.toLong()),
+            MessageLoggerListResult::class
+        ).run {
+            return messages!!
+        }
+    }
+
    override fun getMessageLoggerMessage(conversationId: String, id: Long): ByteArray? {
        sendMessage(
            BridgeMessageType.MESSAGE_LOGGER_REQUEST,
--- a/app/src/main/kotlin/me/rhunk/snapenhance/bridge/common/BridgeMessageType.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/bridge/common/BridgeMessageType.kt
@@ -12,7 +12,8 @@ enum class BridgeMessageType(
    LOCALE_REQUEST(4),
    LOCALE_RESULT(5),
    MESSAGE_LOGGER_REQUEST(6),
-    MESSAGE_LOGGER_RESULT(7);
+    MESSAGE_LOGGER_RESULT(7),
+    MESSAGE_LOGGER_LIST_RESULT(8);

    companion object {
        fun fromValue(value: Int): BridgeMessageType {
--- a/app/src/main/kotlin/me/rhunk/snapenhance/bridge/common/impl/messagelogger/MessageLoggerListResult.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/bridge/common/impl/messagelogger/MessageLoggerListResult.kt
@@ -0,0 +1,18 @@
+package me.rhunk.snapenhance.bridge.common.impl.messagelogger
+
+import android.os.Bundle
+import me.rhunk.snapenhance.bridge.common.BridgeMessage
+
+
+class MessageLoggerListResult(
+    var messages: List<Long>? = null
+) : BridgeMessage() {
+
+    override fun write(bundle: Bundle) {
+        bundle.putLongArray("messages", messages!!.map { it }.toLongArray())
+    }
+
+    override fun read(bundle: Bundle) {
+        messages = bundle.getLongArray("messages")?.toList() ?: emptyList()
+    }
+}
--- a/app/src/main/kotlin/me/rhunk/snapenhance/bridge/common/impl/messagelogger/MessageLoggerRequest.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/bridge/common/impl/messagelogger/MessageLoggerRequest.kt
@@ -6,21 +6,21 @@ import me.rhunk.snapenhance.bridge.common.BridgeMessage
 class MessageLoggerRequest(
    var action: Action? = null,
    var conversationId: String? = null,
-    var messageId: Long? = null,
+    var index: Long? = null,
    var message: ByteArray? = null
 ) : BridgeMessage(){

    override fun write(bundle: Bundle) {
        bundle.putString("action", action!!.name)
        bundle.putString("conversationId", conversationId)
-        bundle.putLong("messageId", messageId ?: 0)
+        bundle.putLong("messageId", index ?: 0)
        bundle.putByteArray("message", message)
    }

    override fun read(bundle: Bundle) {
        action = Action.valueOf(bundle.getString("action")!!)
        conversationId = bundle.getString("conversationId")
-        messageId = bundle.getLong("messageId")
+        index = bundle.getLong("messageId")
        message = bundle.getByteArray("message")
    }

@@ -28,6 +28,7 @@ class MessageLoggerRequest(
        ADD,
        GET,
        CLEAR,
-        DELETE
+        DELETE,
+        LIST_IDS
    }
 }
--- a/app/src/main/kotlin/me/rhunk/snapenhance/bridge/service/BridgeService.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/bridge/service/BridgeService.kt
@@ -17,6 +17,7 @@ import me.rhunk.snapenhance.bridge.common.impl.file.FileAccessRequest
 import me.rhunk.snapenhance.bridge.common.impl.file.FileAccessResult
 import me.rhunk.snapenhance.bridge.common.impl.locale.LocaleRequest
 import me.rhunk.snapenhance.bridge.common.impl.locale.LocaleResult
+import me.rhunk.snapenhance.bridge.common.impl.messagelogger.MessageLoggerListResult
 import me.rhunk.snapenhance.bridge.common.impl.messagelogger.MessageLoggerRequest
 import me.rhunk.snapenhance.bridge.common.impl.messagelogger.MessageLoggerResult
 import java.io.File
@@ -82,7 +83,7 @@ class BridgeService : Service() {
    private fun handleMessageLoggerRequest(msg: MessageLoggerRequest, reply: (Message) -> Unit) {
        when (msg.action) {
            MessageLoggerRequest.Action.ADD  -> {
-                val isSuccess = messageLoggerWrapper.addMessage(msg.conversationId!!, msg.messageId!!, msg.message!!)
+                val isSuccess = messageLoggerWrapper.addMessage(msg.conversationId!!, msg.index!!, msg.message!!)
                reply(MessageLoggerResult(isSuccess).toMessage(BridgeMessageType.MESSAGE_LOGGER_RESULT.value))
                return
            }
@@ -90,12 +91,17 @@ class BridgeService : Service() {
                messageLoggerWrapper.clearMessages()
            }
            MessageLoggerRequest.Action.DELETE -> {
-                messageLoggerWrapper.deleteMessage(msg.conversationId!!, msg.messageId!!)
+                messageLoggerWrapper.deleteMessage(msg.conversationId!!, msg.index!!)
            }
            MessageLoggerRequest.Action.GET -> {
-                val (state, messageData) = messageLoggerWrapper.getMessage(msg.conversationId!!, msg.messageId!!)
+                val (state, messageData) = messageLoggerWrapper.getMessage(msg.conversationId!!, msg.index!!)
                reply(MessageLoggerResult(state, messageData).toMessage(BridgeMessageType.MESSAGE_LOGGER_RESULT.value))
            }
+            MessageLoggerRequest.Action.LIST_IDS -> {
+                val messageIds = messageLoggerWrapper.getMessageIds(msg.conversationId!!, msg.index!!.toInt())
+                reply(MessageLoggerListResult(messageIds).toMessage(BridgeMessageType.MESSAGE_LOGGER_LIST_RESULT.value))
+                return
+            }
            else -> {
                Logger.log(Exception("Unknown message logger action: ${msg.action}"))
            }
@@ -109,8 +115,7 @@ class BridgeService : Service() {
        val compatibleLocale = resources.assets.list("lang")?.find { it.startsWith(deviceLocale) }?.substring(0, 5) ?: "en_US"

        resources.assets.open("lang/$compatibleLocale.json").use { inputStream ->
-            val json = inputStream.bufferedReader().use { it.readText() }
-            reply(LocaleResult(compatibleLocale, json.toByteArray(Charsets.UTF_8)).toMessage(BridgeMessageType.LOCALE_RESULT.value))
+            reply(LocaleResult(compatibleLocale, inputStream.readBytes()).toMessage(BridgeMessageType.LOCALE_RESULT.value))
        }
    }

--- a/app/src/main/kotlin/me/rhunk/snapenhance/database/DatabaseAccess.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/database/DatabaseAccess.kt
@@ -110,6 +110,25 @@ class DatabaseAccess(private val context: ModContext) : Manager {
        }
    }

+    fun getFriendFeed(limit: Int): List<FriendFeedInfo> {
+        return safeDatabaseOperation(openMain()) { database ->
+            val cursor = database.rawQuery(
+                "SELECT * FROM FriendsFeedView ORDER BY _id LIMIT ?",
+                arrayOf(limit.toString())
+            )
+            val list = mutableListOf<FriendFeedInfo>()
+            while (cursor.moveToNext()) {
+                val friendFeedInfo = FriendFeedInfo()
+                try {
+                    friendFeedInfo.write(cursor)
+                } catch (_: Throwable) {}
+                list.add(friendFeedInfo)
+            }
+            cursor.close()
+            list
+        } ?: emptyList()
+    }
+
    fun getConversationMessageFromId(clientMessageId: Long): ConversationMessage? {
        return safeDatabaseOperation(openArroyo()) {
            readDatabaseObject(
--- a/app/src/main/kotlin/me/rhunk/snapenhance/features/impl/spying/MessageLogger.kt
+++ b/app/src/main/kotlin/me/rhunk/snapenhance/features/impl/spying/MessageLogger.kt
@@ -1,6 +1,8 @@
 package me.rhunk.snapenhance.features.impl.spying

+import com.google.gson.JsonObject
 import com.google.gson.JsonParser
+import me.rhunk.snapenhance.Logger
 import me.rhunk.snapenhance.config.ConfigProperty
 import me.rhunk.snapenhance.data.ContentType
 import me.rhunk.snapenhance.data.MessageState
@@ -9,83 +11,114 @@ import me.rhunk.snapenhance.features.Feature
 import me.rhunk.snapenhance.features.FeatureLoadParams
 import me.rhunk.snapenhance.hook.HookStage
 import me.rhunk.snapenhance.hook.Hooker
+import kotlin.time.ExperimentalTime
+import kotlin.time.measureTime
+
+class MessageLogger : Feature("MessageLogger",
+    loadParams = FeatureLoadParams.INIT_SYNC or
+        FeatureLoadParams.ACTIVITY_CREATE_ASYNC
+) {
+    companion object {
+        const val PREFETCH_MESSAGE_COUNT = 20
+        const val PREFETCH_FEED_COUNT = 20
+    }
+
+    //two level of cache to avoid querying the database
+    private val fetchedMessages = mutableListOf<Long>()
+    private val deletedMessageCache = mutableMapOf<Long, JsonObject>()

-class MessageLogger : Feature("MessageLogger", loadParams = FeatureLoadParams.INIT_SYNC or FeatureLoadParams.ACTIVITY_CREATE_ASYNC) {
-    private val messageCache = mutableMapOf<Long, String>()
-    private val removedMessages = linkedSetOf<Long>()
    private val myUserId by lazy { context.database.getMyUserId() }

-    fun isMessageRemoved(messageId: Long) = removedMessages.contains(messageId)
+    fun isMessageRemoved(messageId: Long) = deletedMessageCache.containsKey(messageId)

    fun deleteMessage(conversationId: String, messageId: Long) {
-        messageCache.remove(messageId)
+        fetchedMessages.remove(messageId)
+        deletedMessageCache.remove(messageId)
        context.bridgeClient.deleteMessageLoggerMessage(conversationId, messageId)
    }

+    @OptIn(ExperimentalTime::class)
    override fun asyncOnActivityCreate() {
-        if (!context.database.hasArroyo()) {
-            context.bridgeClient.clearMessageLogger()
+        ConfigProperty.MESSAGE_LOGGER.valueContainer.addPropertyChangeListener {
+            context.config.writeConfig()
+            context.softRestartApp()
        }
+
+        if (!context.database.hasArroyo()) {
+            return
+        }
+
+        measureTime {
+            context.database.getFriendFeed(PREFETCH_FEED_COUNT).forEach { friendFeedInfo ->
+                fetchedMessages.addAll(context.bridgeClient.getLoggedMessageIds(friendFeedInfo.key!!, PREFETCH_MESSAGE_COUNT))
+            }
+        }.also { Logger.debug("Loaded ${fetchedMessages.size} cached messages in $it") }
+    }
+
+    private fun processSnapMessage(messageInstance: Any) {
+        val message = Message(messageInstance)
+
+        if (message.messageState != MessageState.COMMITTED) return
+
+        //exclude messages sent by me
+        if (message.senderId.toString() == myUserId) return
+
+        val messageId = message.messageDescriptor.messageId
+        val conversationId = message.messageDescriptor.conversationId.toString()
+
+        if (message.messageContent.contentType != ContentType.STATUS) {
+            if (fetchedMessages.contains(messageId)) return
+            fetchedMessages.add(messageId)
+
+            context.executeAsync {
+                context.bridgeClient.getMessageLoggerMessage(conversationId, messageId)?.let {
+                    return@executeAsync
+                }
+                context.bridgeClient.addMessageLoggerMessage(conversationId, messageId, context.gson.toJson(messageInstance).toByteArray(Charsets.UTF_8))
+            }
+
+            return
+        }
+
+        //query the deleted message
+        val deletedMessageObject: JsonObject = if (deletedMessageCache.containsKey(messageId))
+            deletedMessageCache[messageId]
+        else {
+            context.bridgeClient.getMessageLoggerMessage(conversationId, messageId)?.let {
+                JsonParser.parseString(it.toString(Charsets.UTF_8)).asJsonObject
+            }
+        } ?: return
+
+        val messageJsonObject = deletedMessageObject.asJsonObject
+
+        //if the message is a snap make it playable
+        if (messageJsonObject["mMessageContent"]?.asJsonObject?.get("mContentType")?.asString == "SNAP") {
+            messageJsonObject["mMetadata"].asJsonObject.addProperty("mPlayableSnapState", "PLAYABLE")
+        }
+
+        //serialize all properties of messageJsonObject and put in the message object
+        messageInstance.javaClass.declaredFields.forEach { field ->
+            field.isAccessible = true
+            messageJsonObject[field.name]?.let { fieldValue ->
+                field.set(messageInstance, context.gson.fromJson(fieldValue, field.type))
+            }
+        }
+
+        //set the message state to PREPARING for visibility
+        with(message.messageContent.contentType) {
+            if (this != ContentType.SNAP && this != ContentType.EXTERNAL_MEDIA) {
+                message.messageState = MessageState.PREPARING
+            }
+        }
+
+        deletedMessageCache[messageId] = deletedMessageObject
    }

-    //FIXME: message disappears when the conversation is set to delete on view
    override fun init() {
        Hooker.hookConstructor(context.classCache.message, HookStage.AFTER, {
            context.config.bool(ConfigProperty.MESSAGE_LOGGER)
-        }) {
-            val message = Message(it.thisObject())
-            val messageId = message.messageDescriptor.messageId
-            val contentType = message.messageContent.contentType
-            val conversationId = message.messageDescriptor.conversationId.toString()
-            val messageState = message.messageState
-
-            if (messageState != MessageState.COMMITTED) return@hookConstructor
-
-            if (contentType == ContentType.STATUS) {
-                //query the deleted message
-                val deletedMessage: String = if (messageCache.containsKey(messageId)) messageCache[messageId] else {
-                    context.bridgeClient.getMessageLoggerMessage(conversationId, messageId)?.toString(Charsets.UTF_8)
-                } ?: return@hookConstructor
-
-                val messageJsonObject = JsonParser.parseString(deletedMessage).asJsonObject
-
-                //if the message is a snap make it playable
-                if (messageJsonObject["mMessageContent"]?.asJsonObject?.get("mContentType")?.asString == "SNAP") {
-                    messageJsonObject["mMetadata"].asJsonObject.addProperty("mPlayableSnapState", "PLAYABLE")
-                }
-
-                //serialize all properties of messageJsonObject and put in the message object
-                message.instanceNonNull().javaClass.declaredFields.forEach { field ->
-                    field.isAccessible = true
-                    val fieldName = field.name
-                    val fieldValue = messageJsonObject[fieldName]
-                    if (fieldValue != null) {
-                        field.set(message.instanceNonNull(), context.gson.fromJson(fieldValue, field.type))
-                    }
-                }
-
-                //set the message state to PREPARING for visibility
-                if (message.messageContent.contentType != ContentType.SNAP && message.messageContent.contentType != ContentType.EXTERNAL_MEDIA) {
-                    message.messageState = MessageState.PREPARING
-                }
-                removedMessages.add(messageId)
-                return@hookConstructor
-            }
-
-            //exclude messages sent by me
-            if (message.senderId.toString() == myUserId) return@hookConstructor
-
-            if (!messageCache.containsKey(messageId)) {
-                context.executeAsync {
-                    val storedMessage = context.bridgeClient.getMessageLoggerMessage(conversationId, messageId)?.toString(Charsets.UTF_8)
-                    if (storedMessage == null) {
-                        messageCache[messageId] = context.gson.toJson(message.instanceNonNull())
-                        context.bridgeClient.addMessageLoggerMessage(conversationId, messageId, messageCache[messageId]!!.toByteArray(Charsets.UTF_8))
-                        return@executeAsync
-                    }
-                    messageCache[messageId] = storedMessage
-                }
-            }
+        }) { param ->
+            processSnapMessage(param.thisObject())
        }
    }
 }