Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ import androidx.compose.ui.unit.sp
import com.openmausbot.companion.audio.VoiceNoteController
import com.openmausbot.companion.core.Chat
import com.openmausbot.companion.core.AttachedMessageContent
import com.openmausbot.companion.core.attachedFiles
import com.openmausbot.companion.core.generatedImages
import com.openmausbot.companion.core.voiceNotes
import com.openmausbot.companion.core.DisplayedMessageAttachment
Expand Down Expand Up @@ -546,6 +547,14 @@ private fun TextBubble(
message.generatedImages.forEach { attachment ->
SharedAttachmentView(threadId, message, attachment, openAttachment)
}
// Documents, audio and video a bot sent with attach_file (MOCA-155).
// The card opens the file sheet; Open hands video to the player.
message.attachedFiles.forEach { attachment ->
SharedAttachmentView(
threadId, message, attachment, openAttachment,
foreground = if (mine) BubbleColor.mineText else MaterialTheme.colorScheme.onSurface,
)
}
// Bots get markdown, you do not — the same split the desktop makes.
// Markdown you did not intend is worse than markdown you did: a
// message about `**` should show the asterisks.
Expand Down Expand Up @@ -597,16 +606,19 @@ private fun SharedAttachmentView(
message: Message,
attachment: DisplayedMessageAttachment,
onOpen: ((DisplayedMessageAttachment, Message, DownloadedFile?) -> Unit)?,
/** Your own bubble is blue with white text; a bot's file sits on the theme surface. */
foreground: Color = BubbleColor.mineText,
) {
if (attachment.kind == DisplayedMessageAttachment.Kind.IMAGE) {
SharedImageAttachment(threadId, message, attachment, onOpen)
return
}
val family = attachment.fileFamily
Row(
modifier = Modifier
.widthIn(max = 360.dp)
.clip(RoundedCornerShape(16.dp))
.background(BubbleColor.mineText.copy(alpha = 0.10f))
.background(foreground.copy(alpha = 0.10f))
.clickable(enabled = onOpen != null, role = Role.Button) {
onOpen?.invoke(attachment, message, null)
}
Expand All @@ -617,17 +629,27 @@ private fun SharedAttachmentView(
horizontalArrangement = Arrangement.spacedBy(10.dp),
verticalAlignment = Alignment.CenterVertically,
) {
Text(stringResource(R.string.mobile_file_b4915d3a), fontSize = 11.sp, fontWeight = FontWeight.Bold, color = BubbleColor.mineText.copy(alpha = 0.68f))
val label = when (family) {
DisplayedMessageAttachment.FileFamily.VIDEO -> R.string.mobile_file_video
DisplayedMessageAttachment.FileFamily.AUDIO -> R.string.mobile_file_audio
DisplayedMessageAttachment.FileFamily.DOCUMENT -> R.string.mobile_file_b4915d3a
}
val hint = if (family == DisplayedMessageAttachment.FileFamily.DOCUMENT) {
R.string.mobile_tap_to_preview_fa5ce0ea
} else {
R.string.mobile_tap_to_play
}
Text(stringResource(label), fontSize = 11.sp, fontWeight = FontWeight.Bold, color = foreground.copy(alpha = 0.68f))
Column(modifier = Modifier.weight(1f)) {
Text(
attachment.name,
fontSize = 14.sp,
fontWeight = FontWeight.Medium,
color = BubbleColor.mineText,
color = foreground,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
Text(stringResource(R.string.mobile_tap_to_preview_fa5ce0ea), fontSize = 12.sp, color = BubbleColor.mineText.copy(alpha = 0.68f))
Text(stringResource(hint), fontSize = 12.sp, color = foreground.copy(alpha = 0.68f))
}
}
}
Expand Down
3 changes: 3 additions & 0 deletions android/app/src/main/res/values-b+zh+Hans/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,9 @@
<string name="mobile_expand_9869e506">展开</string>
<string name="mobile_expanded_6d170474">已展开</string>
<string name="mobile_file_b4915d3a">文件</string>
<string name="mobile_file_video">视频</string>
<string name="mobile_file_audio">音频</string>
<string name="mobile_tap_to_play">点一下以播放</string>
<string name="mobile_for_example_summarize_this_and_lis_12b3d099">例如:整理摘要并列出后续步骤</string>
<string name="mobile_from_5_minutes_to_24_hours_a6bafa40">从 5 分钟到 24 小时</string>
<string name="mobile_group_name_optional_0fd3a15c">群组名称(选填)</string>
Expand Down
3 changes: 3 additions & 0 deletions android/app/src/main/res/values-b+zh+Hant/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,9 @@
<string name="mobile_expand_9869e506">展開</string>
<string name="mobile_expanded_6d170474">已展開</string>
<string name="mobile_file_b4915d3a">檔案</string>
<string name="mobile_file_video">影片</string>
<string name="mobile_file_audio">音訊</string>
<string name="mobile_tap_to_play">點一下以播放</string>
<string name="mobile_for_example_summarize_this_and_lis_12b3d099">例如:整理摘要並列出後續步驟</string>
<string name="mobile_from_5_minutes_to_24_hours_a6bafa40">從 5 分鐘到 24 小時</string>
<string name="mobile_group_name_optional_0fd3a15c">群組名稱(選填)</string>
Expand Down
3 changes: 3 additions & 0 deletions android/app/src/main/res/values/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,9 @@
<string name="mobile_expand_9869e506">Expand</string>
<string name="mobile_expanded_6d170474">Expanded</string>
<string name="mobile_file_b4915d3a">FILE</string>
<string name="mobile_file_video">VIDEO</string>
<string name="mobile_file_audio">AUDIO</string>
<string name="mobile_tap_to_play">Tap to play</string>
<string name="mobile_for_example_summarize_this_and_lis_12b3d099">For example: summarize this and list the next steps</string>
<string name="mobile_from_5_minutes_to_24_hours_a6bafa40">From 5 minutes to 24 hours</string>
<string name="mobile_group_name_optional_0fd3a15c">Group name (optional)</string>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,20 @@ data class DisplayedMessageAttachment(
val durationMs: Double? = null,
) {
enum class Kind { IMAGE, FILE, AUDIO }

/**
* What a file card promises: video and audio play, the rest preview. Read
* from the extension, the signal the server uses to pick the file's type
* (mimeFor in server/message-file.ts). Port of the iOS `fileFamily`.
*/
enum class FileFamily { VIDEO, AUDIO, DOCUMENT }

val fileFamily: FileFamily
get() = when (name.substringAfterLast('.', "").lowercase()) {
"mp4", "mov", "m4v", "webm" -> FileFamily.VIDEO
"mp3", "m4a", "wav", "aac", "ogg", "oga", "opus", "flac" -> FileFamily.AUDIO
else -> FileFamily.DOCUMENT
}
}

data class AttachedMessageContent(
Expand Down Expand Up @@ -212,6 +226,25 @@ data class AttachedMessageContent(
}
}

/**
* Files a bot sent with attach_file — documents, audio and video — in wire
* order and deduplicated like [generatedImages]. Before these rendered, a
* file-only reply was an empty bubble on the phone (MOCA-155). The path goes
* unchanged to the originating message's authenticated file route.
*/
val Message.attachedFiles: List<DisplayedMessageAttachment>
get() = attachments.orEmpty()
.filter { it.kind == "file" && !it.path.isNullOrBlank() }
.distinctBy { it.path }
.map {
val path = requireNotNull(it.path)
DisplayedMessageAttachment(
kind = DisplayedMessageAttachment.Kind.FILE,
path = path,
name = AttachedMessageContent.displayName(it.name, path, DisplayedMessageAttachment.Kind.FILE),
)
}

/** Preserve the server path exactly for the originating message's authenticated file route. */
val Message.generatedImages: List<DisplayedMessageAttachment>
get() = attachments.orEmpty()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,10 @@ fun rosterPreview(messages: List<Message>, detail: ActivityDetail): String =

/** What a single message reads as in a roster row. */
internal fun previewText(message: Message): String = when (message.kind) {
Message.Kind.TEXT -> message.webhookContent?.task ?: message.text.orEmpty()
Message.Kind.TEXT -> message.webhookContent?.task
?: message.text?.takeIf { it.isNotEmpty() }
// A bot that only sent a file says so by its name.
?: message.attachedFiles.firstOrNull()?.name.orEmpty()
// a pending card's question is the preview; the roster row already says
// "waiting on you" beside it
Message.Kind.OPTIONS -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1532,12 +1532,18 @@ data class ServerSession(val id: String, val label: String, val scopes: List<Str
@Serializable
data class ServerEnvironment(val environmentId: String, val label: String)

/** Unknown attachment kinds remain decodable and are not rendered. */
/**
* Image entries display inline, file entries (a bot's attach_file: documents,
* audio, video) show as file cards ([attachedFiles]), audio entries render as
* voice notes. Unknown attachment kinds remain decodable and are not rendered.
*/
@Serializable
data class MessageImageAttachment(
val kind: String,
val path: String? = null,
val mime: String? = null,
/** `kind == "audio"`: the server's duration estimate, used until the player loads metadata. */
val durationMs: Double? = null,
/** The file's name as the bot sent it, for `kind == "file"`. Presentation only; basenamed before display. */
val name: String? = null,
)
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,47 @@ class GeneratedImageTest {
assertEquals(emptyList(), message.generatedImages)
assertEquals("Still visible", message.text)
}

/**
* A bot's attach_file sends documents, audio and video as `kind:"file"`
* with a name (server/bot-attachment.ts). The phone dropped them, and the
* message read as an empty bubble (MOCA-155). Port of the iOS test.
*/
@Test
fun aBotsFileAttachmentsBecomeNamedFileCards() {
val source = """{"id":"sent","role":"bot","kind":"text","at":1,"text":"","attachments":[{"kind":"file","path":"/data/attachments/9f.mp4","mime":"video/mp4","name":"demo clip.mp4"},{"kind":"image","path":"/data/attachments/a1.png","mime":"image/png"},{"kind":"file","path":"/data/attachments/77.pdf","mime":"application/pdf","name":"../../report.pdf"},{"kind":"file","path":"/data/attachments/9f.mp4","name":"again.mp4"},{"kind":"file","path":" "},{"kind":"hologram","path":"/data/attachments/x.bin"}]}"""
val message = CompanionJson.decodeFromString<Message>(source)
assertEquals(
listOf(
DisplayedMessageAttachment(DisplayedMessageAttachment.Kind.FILE, "demo clip.mp4", "/data/attachments/9f.mp4"),
// A crafted name is presentation only and never a path.
DisplayedMessageAttachment(DisplayedMessageAttachment.Kind.FILE, "report.pdf", "/data/attachments/77.pdf"),
),
message.attachedFiles,
)
assertEquals(listOf("/data/attachments/a1.png"), message.generatedImages.map { it.path }, "images keep their own card")
val cached = CompanionJson.decodeFromString<Message>(CompanionJson.encodeToString(message))
assertEquals(message.attachedFiles, cached.attachedFiles)
}

@Test
fun aFileOnlyBotMessagePreviewsAsItsFileName() {
val message = CompanionJson.decodeFromString<Message>(
"""{"id":"sent","role":"bot","kind":"text","at":1,"text":"","attachments":[{"kind":"file","path":"/data/attachments/9f.mp4","mime":"video/mp4","name":"demo clip.mp4"}]}""",
)
assertEquals("demo clip.mp4", rosterPreview(listOf(message), ActivityDetail.FULL))
assertEquals("Here is the clip", rosterPreview(listOf(message.copy(text = "Here is the clip")), ActivityDetail.FULL))
}

@Test
fun aFileCardKnowsVideoAndAudioFromTheExtension() {
fun family(name: String) = DisplayedMessageAttachment(DisplayedMessageAttachment.Kind.FILE, name, "/data/attachments/x").fileFamily
assertEquals(DisplayedMessageAttachment.FileFamily.VIDEO, family("demo clip.MP4"))
assertEquals(DisplayedMessageAttachment.FileFamily.VIDEO, family("screen.mov"))
assertEquals(DisplayedMessageAttachment.FileFamily.VIDEO, family("talk.webm"))
assertEquals(DisplayedMessageAttachment.FileFamily.AUDIO, family("memo.m4a"))
assertEquals(DisplayedMessageAttachment.FileFamily.AUDIO, family("song.mp3"))
assertEquals(DisplayedMessageAttachment.FileFamily.DOCUMENT, family("weekly-report.pdf"))
assertEquals(DisplayedMessageAttachment.FileFamily.DOCUMENT, family("notes"))
}
}
Binary file added docs/screenshots/moca-155-bot-files-ios.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
12 changes: 10 additions & 2 deletions ios/App/AttachmentViews.swift
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,18 @@ struct TranscriptAttachmentView: View {
.accessibilityHint(thumbnail == nil ? "Loads the image preview" : "Opens the image full screen")
}

private var fileSymbol: String {
switch attachment.fileFamily {
case .video: "play.rectangle.fill"
case .audio: "waveform"
case .document: "doc.fill"
}
}

private var fileCard: some View {
Button(action: openPreview) {
HStack(spacing: 10) {
Image(systemName: "doc.fill")
Image(systemName: fileSymbol)
.font(.system(size: 16, weight: .medium))
.foregroundStyle(foreground)
.frame(width: 38, height: 38)
Expand All @@ -248,7 +256,7 @@ struct TranscriptAttachmentView: View {
Text(attachment.name)
.font(.system(size: 13, weight: .semibold))
.lineLimit(1)
Text(previewLoading ? "Opening…" : "Tap to preview")
Text(previewLoading ? "Opening…" : attachment.fileFamily == .document ? "Tap to preview" : "Tap to play")
.font(.system(size: 11))
.foregroundStyle(foreground.opacity(0.68))
}
Expand Down
8 changes: 8 additions & 0 deletions ios/App/ChatView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1883,6 +1883,14 @@ struct TextBubble: View {
messageId: message.id, foreground: mine ? BubbleColor.mineText : .primary
)
}
// Documents, audio and video a bot sent with attach_file. The
// card opens the full-screen viewer, which plays video.
ForEach(message.attachedFiles, id: \.path) { attachment in
TranscriptAttachmentView(
attachment: attachment, threadId: chat.threadId,
messageId: message.id, foreground: mine ? BubbleColor.mineText : .primary
)
}
// Bots get markdown, you do not — the same split the desktop
// makes. Markdown you did not intend is worse than markdown
// you did: a message about `**` should show the asterisks.
Expand Down
10 changes: 10 additions & 0 deletions ios/App/Localizable.xcstrings
Original file line number Diff line number Diff line change
Expand Up @@ -11197,6 +11197,16 @@
}
}
},
"Tap to play": {
"localizations": {
"pt-BR": {
"stringUnit": {
"state": "translated",
"value": "Toque para reproduzir"
}
}
}
},
"Nothing scheduled": {
"localizations": {
"pt-BR": {
Expand Down
33 changes: 33 additions & 0 deletions ios/Sources/CompanionCore/AttachedMessageContent.swift
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,22 @@ public struct DisplayedMessageAttachment: Hashable, Sendable {
self.path = path
self.name = name
}

/// What a file card promises: video and audio play in the viewer, the
/// rest preview. Read from the extension, the same signal the server
/// uses to pick the file's type (mimeFor in server/message-file.ts).
public enum FileFamily: Sendable {
case video, audio, document
}

public var fileFamily: FileFamily {
let ext = (name as NSString).pathExtension.lowercased()
switch ext {
case "mp4", "mov", "m4v", "webm": return .video
case "mp3", "m4a", "wav", "aac", "ogg", "oga", "opus", "flac": return .audio
default: return .document
}
}
}

public struct AttachedMessageContent: Hashable, Sendable {
Expand Down Expand Up @@ -294,6 +310,23 @@ public struct AttachedMessageContent: Hashable, Sendable {
}

extension Message {
/// Files a bot sent with attach_file — documents, audio and video — in wire
/// order and deduplicated like generatedImages. Before these rendered, a
/// file-only reply was an empty bubble on the phone (MOCA-155). Paths are
/// server metadata, passed unchanged to the message-scoped file route.
public var attachedFiles: [DisplayedMessageAttachment] {
var seen = Set<String>()
return (attachments ?? []).compactMap { attachment in
guard attachment.kind == "file", let path = attachment.path,
!path.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty,
seen.insert(path).inserted else { return nil }
return DisplayedMessageAttachment(
kind: .file, path: path,
name: AttachedMessageContent.displayName(providedName: attachment.name, path: path, kind: .file)
)
}
}

/// Paths are server metadata, passed unchanged to the message-scoped file route.
public var generatedImages: [DisplayedMessageAttachment] {
var seen = Set<String>()
Expand Down
6 changes: 5 additions & 1 deletion ios/Sources/CompanionCore/ChatPreferences.swift
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,11 @@ public func rosterPreview(_ messages: [Message], detail: ActivityDetail) -> Stri
/// What a single message reads as in a roster row.
func previewText(of message: Message) -> String {
switch message.kind {
case .text: return message.webhookContent?.task ?? message.text ?? ""
case .text:
if let task = message.webhookContent?.task { return task }
if let text = message.text, !text.isEmpty { return text }
// A bot that only sent a file says so by its name.
return message.attachedFiles.first?.name ?? ""
// a pending card's question is the preview; the roster row already
// says "waiting on you" beside it
case .options:
Expand Down
12 changes: 9 additions & 3 deletions ios/Sources/CompanionCore/Models.swift
Original file line number Diff line number Diff line change
Expand Up @@ -1708,13 +1708,19 @@ public struct ServerEnvironment: Codable, Hashable, Sendable {
public var version: String?
}

/// Keep future attachment kinds decodable; image entries display inline and
/// audio entries render as voice notes (Message.voiceNotes). Unknown kinds
/// decode without breaking, so a newer computer never gaps the transcript.
/// Keep future attachment kinds decodable; image entries display inline,
/// file entries (a bot's attach_file: documents, audio, video) show as file
/// cards (Message.attachedFiles), and audio entries render as voice notes
/// (Message.voiceNotes). Unknown kinds decode without breaking, so a newer
/// computer never gaps the transcript.
public struct MessageImageAttachment: Codable, Hashable, Sendable {
public var kind: String
public var path: String?
public var mime: String?
/// The file's name as the bot sent it, for a `kind: "file"` entry.
/// Presentation only: it is basenamed before display and never used as
/// a path.
public var name: String?
/// The server's duration estimate for an audio attachment, in
/// milliseconds; shown until the player loads real metadata.
public var durationMs: Double?
Expand Down
Loading
Loading