majjo/XSnap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(experimental): audio call recorder

Browse Source
This commit is contained in:
rhunk
2024-03-08 17:49:53 +01:00
parent ec12980571
commit db2cc0d785
12 changed files with 287 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
app/src/main/kotlin/me/rhunk/snapenhance/download/DownloadProcessor.kt
View File

@@ -254,7 +254,7 @@ class DownloadProcessor (
val outputFile = File.createTempFile("voice_note", ".$format")
newFFMpegProcessor(pendingTask).execute(FFMpegProcessor.Request(
action = FFMpegProcessor.Action.AUDIO_CONVERSION,
inputs = listOf(media.file),
inputs = listOf(media.file.absolutePath),
output = outputFile
))
media.file.delete()
@@ -291,7 +291,7 @@ class DownloadProcessor (
runCatching {
newFFMpegProcessor(pendingTask).execute(FFMpegProcessor.Request(
action = FFMpegProcessor.Action.DOWNLOAD_DASH,
inputs = listOf(dashPlaylistFile),
inputs = listOf(dashPlaylistFile.absolutePath),
output = outputFile,
startTime = dashOptions.offsetTime,
duration = dashOptions.duration
@@ -357,6 +357,22 @@ class DownloadProcessor (
}
runCatching {
if (downloadRequest.isAudioStream) {
val streamUrl = downloadRequest.inputMedias.first().content
val outputFile = File.createTempFile("audio_stream", ".mp3")
callbackOnProgress("Downloading audio stream")
pendingTask.updateProgress("Downloading audio stream")
newFFMpegProcessor(pendingTask).execute(FFMpegProcessor.Request(
action = FFMpegProcessor.Action.DOWNLOAD_AUDIO_STREAM,
inputs = listOf(streamUrl),
output = outputFile,
audioStreamFormat = downloadRequest.audioStreamFormat
))
saveMediaToGallery(pendingTask, outputFile, downloadMetadata)
return@launch
}
//first download all input medias into cache
val downloadedMedias = downloadInputMedias(pendingTask, downloadRequest).map {
it.key to DownloadedFile(it.value, FileType.fromFile(it.value))
@@ -406,7 +422,7 @@ class DownloadProcessor (
newFFMpegProcessor(pendingTask).execute(FFMpegProcessor.Request(
action = FFMpegProcessor.Action.MERGE_OVERLAY,
inputs = listOf(renamedMedia),
inputs = listOf(renamedMedia.absolutePath),
output = mergedOverlay,
overlay = renamedOverlayMedia
))

30
app/src/main/kotlin/me/rhunk/snapenhance/download/FFMpegProcessor.kt
View File

@@ -1,5 +1,6 @@
package me.rhunk.snapenhance.download
import android.media.AudioFormat
import android.media.MediaMetadataRetriever
import com.arthenica.ffmpegkit.FFmpegKit
import com.arthenica.ffmpegkit.FFmpegSession
@@ -9,6 +10,7 @@ import kotlinx.coroutines.suspendCancellableCoroutine
import me.rhunk.snapenhance.LogManager
import me.rhunk.snapenhance.RemoteSideContext
import me.rhunk.snapenhance.common.config.impl.DownloaderConfig
import me.rhunk.snapenhance.common.data.download.AudioStreamFormat
import me.rhunk.snapenhance.common.logger.LogLevel
import me.rhunk.snapenhance.task.PendingTask
import java.io.File
@@ -62,16 +64,18 @@ class FFMpegProcessor(
DOWNLOAD_DASH,
MERGE_OVERLAY,
AUDIO_CONVERSION,
MERGE_MEDIA
MERGE_MEDIA,
DOWNLOAD_AUDIO_STREAM,
}
data class Request(
val action: Action,
val inputs: List<File>,
val inputs: List<String>,
val output: File,
val overlay: File? = null, //only for MERGE_OVERLAY
val startTime: Long? = null, //only for DOWNLOAD_DASH
val duration: Long? = null //only for DOWNLOAD_DASH
val duration: Long? = null, //only for DOWNLOAD_DASH
val audioStreamFormat: AudioStreamFormat? = null, //only for DOWNLOAD_AUDIO_STREAM
)
@@ -113,8 +117,8 @@ class FFMpegProcessor(
}
val inputArguments = ArgumentList().apply {
args.inputs.forEach { file ->
this += "-i" to file.absolutePath
args.inputs.forEach { path ->
this += "-i" to path
}
}
@@ -150,7 +154,7 @@ class FFMpegProcessor(
inputArguments.clear()
val filesInfo = args.inputs.mapNotNull { file ->
runCatching {
MediaMetadataRetriever().apply { setDataSource(file.absolutePath) }
MediaMetadataRetriever().apply { setDataSource(file) }
}.getOrNull()?.let { file to it }
}
@@ -173,7 +177,7 @@ class FFMpegProcessor(
containsNoSound = true
filterSecondPart.append("[v$index][${filesInfo.size}]")
}
inputArguments += "-i" to file.absolutePath
inputArguments += "-i" to file
}
if (containsNoSound) {
@@ -194,6 +198,18 @@ class FFMpegProcessor(
filesInfo.forEach { it.second.close() }
}
Action.DOWNLOAD_AUDIO_STREAM -> {
outputArguments.clear()
globalArguments += "-f" to when (args.audioStreamFormat!!.encoding) {
AudioFormat.ENCODING_PCM_8BIT -> "u8"
AudioFormat.ENCODING_PCM_16BIT -> "s16le"
AudioFormat.ENCODING_PCM_FLOAT -> "f32le"
AudioFormat.ENCODING_PCM_32BIT -> "s32le"
else -> throw IllegalArgumentException("Unsupported audio encoding")
}
globalArguments += "-ar" to args.audioStreamFormat!!.sampleRate.toString()
globalArguments += "-ac" to args.audioStreamFormat!!.channels.toString()
}
}
outputArguments += args.output.absolutePath
newFFMpegTask(globalArguments, inputArguments, outputArguments)

2
app/src/main/kotlin/me/rhunk/snapenhance/ui/manager/pages/TasksRoot.kt
View File

@@ -101,7 +101,7 @@ class TasksRoot : Routes.Route() {
runCatching {
context.shortToast("Merging ${filesToMerge.size} files")
FFMpegProcessor.newFFMpegProcessor(context, pendingTask).execute(
FFMpegProcessor.Request(FFMpegProcessor.Action.MERGE_MEDIA, filesToMerge, mergedFile)
FFMpegProcessor.Request(FFMpegProcessor.Action.MERGE_MEDIA, filesToMerge.map { it.absolutePath }, mergedFile)
)
DownloadProcessor(context, object: DownloadCallback.Default() {
override fun onSuccess(outputPath: String) {

7
common/src/main/assets/lang/en_US.json
View File

@@ -686,6 +686,10 @@
"name": "Story Logger",
"description": "Provides a history of friends stories"
},
"call_recorder": {
"name": "Call Recorder",
"description": "Automatically records audio calls"
},
"app_passcode": {
"name": "App Passcode",
"description": "Sets a passcode to lock the app"
@@ -969,7 +973,8 @@
"profile_picture": "Profile Picture",
"story_logger": "Story Logger",
"message_logger": "Message Logger",
"merged": "Merged"
"merged": "Merged",
"voice_call": "Voice Call"
},
"chat_action_menu": {

1
common/src/main/kotlin/me/rhunk/snapenhance/common/config/impl/Experimental.kt
View File

@@ -24,6 +24,7 @@ class Experimental : ConfigContainer() {
val convertMessageLocally = boolean("convert_message_locally") { requireRestart() }
val newChatActionMenu = boolean("new_chat_action_menu") { requireRestart() }
val storyLogger = boolean("story_logger") { requireRestart(); addNotices(FeatureNotice.UNSTABLE); }
val callRecorder = boolean("call_recorder") { requireRestart(); addNotices(FeatureNotice.UNSTABLE); }
val appPasscode = string("app_passcode")
val appLockOnResume = boolean("app_lock_on_resume")
val infiniteStoryBoost = boolean("infinite_story_boost")

11
common/src/main/kotlin/me/rhunk/snapenhance/common/data/download/DownloadRequest.kt
View File

@@ -6,6 +6,8 @@ import java.util.Locale
data class DashOptions(val offsetTime: Long, val duration: Long?)
data class AudioStreamFormat(val channels: Int, val sampleRate: Int, val encoding: Int)
data class InputMedia(
val content: String,
val type: DownloadMediaType,
@@ -17,18 +19,23 @@ data class InputMedia(
class DownloadRequest(
val inputMedias: Array<InputMedia>,
val dashOptions: DashOptions? = null,
val audioStreamFormat: AudioStreamFormat? = null,
private val flags: Int = 0,
) {
object Flags {
const val MERGE_OVERLAY = 1
const val IS_DASH_PLAYLIST = 2
const val DASH_PLAYLIST = 2
const val AUDIO_STREAM = 4
}
val isDashPlaylist: Boolean
get() = flags and Flags.IS_DASH_PLAYLIST != 0
get() = flags and Flags.DASH_PLAYLIST != 0
val shouldMergeOverlay: Boolean
get() = flags and Flags.MERGE_OVERLAY != 0
val isAudioStream: Boolean
get() = flags and Flags.AUDIO_STREAM != 0
}
fun String.sanitizeForPath(): String {

3
common/src/main/kotlin/me/rhunk/snapenhance/common/data/download/MediaDownloadSource.kt
View File

@@ -16,7 +16,8 @@ enum class MediaDownloadSource(
PROFILE_PICTURE("profile_picture", "profile_picture"),
STORY_LOGGER("story_logger", "story_logger"),
MESSAGE_LOGGER("message_logger", "message_logger"),
MERGED("merged", "merged");
MERGED("merged", "merged"),
VOICE_CALL("voice_call", "voice_call");
fun matches(source: String?): Boolean {
if (source == null) return false

20
core/src/main/kotlin/me/rhunk/snapenhance/core/DownloadManagerClient.kt
View File

@@ -31,7 +31,7 @@ class DownloadManagerClient (
)
),
dashOptions = DashOptions(offsetTime, duration),
flags = DownloadRequest.Flags.IS_DASH_PLAYLIST
flags = DownloadRequest.Flags.DASH_PLAYLIST
)
)
}
@@ -67,4 +67,22 @@ class DownloadManagerClient (
)
)
}
fun downloadStream(
streamUrl: String,
audioStreamFormat: AudioStreamFormat
) {
enqueueDownloadRequest(
DownloadRequest(
inputMedias = arrayOf(
InputMedia(
content = streamUrl,
type = DownloadMediaType.REMOTE_MEDIA
)
),
flags = DownloadRequest.Flags.AUDIO_STREAM,
audioStreamFormat = audioStreamFormat
)
)
}
}

2
core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/downloader/MediaDownloader.kt
View File

@@ -76,7 +76,7 @@ class MediaDownloader : MessagingRuleFeature("MediaDownloader", MessagingRuleTyp
context.translation.getCategory("download_processor")
}
private fun provideDownloadManagerClient(
fun provideDownloadManagerClient(
mediaIdentifier: String,
mediaAuthor: String,
creationTimestamp: Long? = null,

133
core/src/main/kotlin/me/rhunk/snapenhance/core/features/impl/experiments/CallRecorder.kt Normal file
View File

@@ -0,0 +1,133 @@
package me.rhunk.snapenhance.core.features.impl.experiments
import android.media.AudioAttributes
import android.media.AudioFormat
import android.media.AudioTrack
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeoutOrNull
import me.rhunk.snapenhance.common.data.download.AudioStreamFormat
import me.rhunk.snapenhance.common.data.download.MediaDownloadSource
import me.rhunk.snapenhance.core.features.Feature
import me.rhunk.snapenhance.core.features.FeatureLoadParams
import me.rhunk.snapenhance.core.features.impl.downloader.MediaDownloader
import me.rhunk.snapenhance.core.util.hook.HookStage
import me.rhunk.snapenhance.core.util.hook.hook
import me.rhunk.snapenhance.core.util.hook.hookConstructor
import me.rhunk.snapenhance.core.util.ktx.getObjectField
import me.rhunk.snapenhance.core.util.ktx.getObjectFieldOrNull
import me.rhunk.snapenhance.core.util.media.HttpServer
import java.io.PipedInputStream
import java.io.PipedOutputStream
import java.nio.ByteBuffer
import java.util.UUID
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
class CallRecorder : Feature("Call Recorder", loadParams = FeatureLoadParams.INIT_SYNC) {
private val httpServer = HttpServer(
timeout = Integer.MAX_VALUE
)
override fun init() {
if (!context.config.experimental.callRecorder.get()) return
val streamHandlers = ConcurrentHashMap<Int, MutableList<(data: ByteArray) -> Unit>>() // audioTrack -> handlers
val participants = CopyOnWriteArrayList<String>()
findClass("com.snapchat.talkcorev3.CallingSessionState").hookConstructor(HookStage.AFTER) { param ->
val instance = param.thisObject<Any>()
val callingState = instance.getObjectFieldOrNull("mLocalUser")?.getObjectField("mCallingState")
if (callingState.toString() == "IN_CALL") {
participants.clear()
participants.addAll((instance.getObjectField("mParticipants") as Map<*, *>).keys.map { it.toString() })
}
}
AudioTrack::class.java.apply {
getConstructor(
AudioAttributes::class.java,
AudioFormat::class.java,
Int::class.javaPrimitiveType,
Int::class.javaPrimitiveType,
Int::class.javaPrimitiveType,
).hook(HookStage.BEFORE) { param ->
val audioAttributes = param.arg<AudioAttributes>(0)
if (audioAttributes.usage != AudioAttributes.USAGE_VOICE_COMMUNICATION) return@hook
val audioFormat = param.arg<AudioFormat>(1)
val hashCode = param.thisObject<Any>().hashCode()
lateinit var streamUrl: String
streamUrl = httpServer.ensureServerStarted()?.putContent(
object: HttpServer.HttpContent() {
override val contentType: String = "audio/wav"
override val chunked: Boolean = true
override val contentLength: Long? = null
override val newBody: () -> HttpServer.HttpBody = {
object: HttpServer.HttpBody() {
val outputStream = PipedOutputStream()
val inputStream = PipedInputStream(outputStream)
val handler: (byteArray: ByteArray) -> Unit = handler@{ byteArray ->
if (byteArray.isEmpty()) {
httpServer.removeUrl(streamUrl)
return@handler
}
runCatching {
outputStream.write(byteArray)
outputStream.flush()
}.onFailure {
context.log.warn("Failed to write to streaming url ${it.localizedMessage}")
}
}
override val onOpen: () -> Unit = {
streamHandlers.getOrPut(hashCode) { CopyOnWriteArrayList() }.add(handler)
}
override val readBytes: (byteArray: ByteArray) -> Int = { byteArray ->
runBlocking {
withTimeoutOrNull(3000L) {
inputStream.read(byteArray)
} ?: -1
}
}
override val onClose: () -> Unit = {
context.log.verbose("Streaming url closed")
streamHandlers[hashCode]?.remove(handler)
outputStream.close()
inputStream.close()
}
}
}
}
) ?: return@hook
context.log.verbose("streaming url = $streamUrl, sampleRate = ${audioFormat.sampleRate}, audioFormat = ${audioFormat.encoding}")
context.feature(MediaDownloader::class).provideDownloadManagerClient(
UUID.randomUUID().toString(),
participants.mapNotNull { context.database.getFriendInfo(it)?.mutableUsername }.joinToString("-"),
System.currentTimeMillis(),
MediaDownloadSource.VOICE_CALL
).downloadStream(streamUrl, AudioStreamFormat(audioFormat.channelCount, audioFormat.sampleRate, audioFormat.encoding))
}
getMethod("write", ByteBuffer::class.java, Int::class.javaPrimitiveType, Int::class.javaPrimitiveType).hook(HookStage.BEFORE) { param ->
streamHandlers[param.thisObject<Any>().hashCode()]?.let { handlers ->
val byteBuffer = param.arg<ByteBuffer>(0)
val position = byteBuffer.position()
val buffer = ByteArray(param.arg(1))
byteBuffer.get(buffer)
byteBuffer.position(position)
handlers.forEach { it(buffer) }
}
}
hook("release", HookStage.BEFORE) {
streamHandlers.remove(it.thisObject<Any>().hashCode())?.forEach { it(ByteArray(0)) }
}
}
}
}

1
core/src/main/kotlin/me/rhunk/snapenhance/core/manager/impl/FeatureManager.kt
View File

@@ -120,6 +120,7 @@ class FeatureManager(
DisablePermissionRequests(),
SessionEvents(),
DefaultVolumeControls(),
CallRecorder(),
)
initializeFeatures()

79
core/src/main/kotlin/me/rhunk/snapenhance/core/util/media/HttpServer.kt
View File

@@ -27,7 +27,20 @@ class HttpServer(
private var timeoutJob: Job? = null
private var socketJob: Job? = null
private val cachedData = ConcurrentHashMap<String, Pair<InputStream, Long>>()
abstract class HttpBody {
abstract val readBytes: (byteArray: ByteArray) -> Int
open val onOpen: () -> Unit = {}
open val onClose: () -> Unit = {}
}
abstract class HttpContent {
abstract val contentType: String
abstract val chunked: Boolean
abstract val contentLength: Long?
abstract val newBody: () -> HttpBody
}
private val cachedData = ConcurrentHashMap<String, HttpContent>()
private var serverSocket: ServerSocket? = null
fun ensureServerStarted(): HttpServer? {
@@ -83,15 +96,39 @@ class HttpServer(
}
fun close() {
serverSocket?.close()
runCatching {
serverSocket?.close()
}
}
fun putDownloadableContent(inputStream: InputStream, size: Long): String {
val key = System.nanoTime().toString(16)
cachedData[key] = inputStream to size
cachedData[key] = object : HttpContent() {
override val contentType: String = "application/octet-stream"
override val chunked: Boolean = false
override val contentLength: Long = size
override val newBody: () -> HttpBody = {
object : HttpBody() {
override val readBytes: (byteArray: ByteArray) -> Int = { byteArray ->
inputStream.read(byteArray)
}
}
}
}
return "http://127.0.0.1:$port/$key"
}
fun putContent(httpContent: HttpContent): String {
val key = System.nanoTime().toString(16)
cachedData[key] = httpContent
return "http://127.0.0.1:$port/$key"
}
fun removeUrl(url: String) {
val key = url.substringAfterLast('/')
cachedData.remove(key)
}
private fun handleRequest(socket: Socket) {
val reader = BufferedReader(InputStreamReader(socket.getInputStream()))
val outputStream = socket.getOutputStream()
@@ -138,13 +175,41 @@ class HttpServer(
with(writer) {
println("HTTP/1.1 200 OK")
println("Content-type: " + "application/octet-stream")
println("Content-length: " + requestedData.second)
if (requestedData.chunked) println("Transfer-encoding: chunked")
else println("Content-length: " + requestedData.contentLength)
println()
flush()
}
cachedData.remove(fileRequested)
requestedData.first.use {
it.copyTo(outputStream)
val responseBody = requestedData.newBody()
responseBody.onOpen()
try {
if (requestedData.chunked) {
val buffer = ByteArray(32768)
while (true) {
val read = responseBody.readBytes(buffer)
if (read == -1) break
outputStream.write(Integer.toHexString(read).toByteArray())
outputStream.write("\r\n".toByteArray())
outputStream.write(buffer, 0, read)
outputStream.write("\r\n".toByteArray())
outputStream.flush()
}
} else {
cachedData.remove(fileRequested)
val buffer = ByteArray(4096)
while (true) {
val read = responseBody.readBytes(buffer)
if (read == -1) break
outputStream.write(buffer, 0, read)
outputStream.flush()
}
}
} catch (t: Throwable) {
AbstractLogger.directDebug("failed to write to socket ${t.localizedMessage}")
} finally {
if (requestedData.chunked) runCatching { outputStream.write("0\r\n\r\n".toByteArray()) }
responseBody.onClose()
}
outputStream.flush()
close()