feat: support WebM media fragments in Anki exports for local video

common/anki/anki.ts

@@ -1,5 +1,5 @@
 import { AudioClip } from '@project/common/audio-clip';
-import { AnkiExportMode, CardModel, Image, Progress } from '@project/common';
+import { AnkiExportMode, CardModel, MediaFragment, Progress } from '@project/common';
 import { HttpFetcher, Fetcher } from '@project/common';
 import { AnkiSettings, AnkiSettingsFieldKey } from '@project/common/settings';
 import sanitize from 'sanitize-filename';
@@ -15,6 +15,22 @@ const alphaNumericCharacters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv
 const unsafeURLChars = /[:\/\?#\[\]@!$&'()*+,;= "<>%{}|\\^`]/g;
 const replacement = '_';
 
+const logMediaCreationTime = (type: string, extension: string, durationMs: number, fileName: string) => {
+    console.info(`[asbplayer] ${type} creation took ${durationMs}ms (${fileName}, .${extension})`);
+};
+
+const timedMediaBase64 = async (
+    type: string,
+    extension: string,
+    fileName: string,
+    getBase64: () => Promise<string>
+) => {
+    const startedAt = Date.now();
+    const data = await getBase64();
+    logMediaCreationTime(type, extension, Date.now() - startedAt, fileName);
+    return data;
+};
+
 export function escapeAnkiQuery(query: string) {
     let escaped = '';
 
@@ -136,7 +152,7 @@ export interface ExportParams {
     track3: string | undefined;
     definition: string | undefined;
     audioClip: AudioClip | undefined;
-    image: Image | undefined;
+    image: MediaFragment | undefined;
     word: string | undefined;
     source: string | undefined;
     url: string | undefined;
@@ -146,6 +162,17 @@ export interface ExportParams {
     ankiConnectUrl?: string;
 }
 
+interface EncodedMedia {
+    sanitizedName: string;
+    data: string;
+}
+
+interface Base64Exportable {
+    name: string;
+    extension: string;
+    base64: () => Promise<string>;
+}
+
 export async function exportCard(
     card: CardModel,
     ankiSettings: AnkiSettings,
@@ -166,6 +193,8 @@ export async function exportCard(
                   card.audio.error
               );
 
+    const serializedMediaFragment = card.mediaFragment ?? card.image;
+
     return await anki.export({
         text: card.text ?? extractText(card.subtitle, card.surroundingSubtitles),
         track1: extractText(card.subtitle, card.surroundingSubtitles, 0),
@@ -174,14 +203,14 @@ export async function exportCard(
         definition: card.definition,
         audioClip,
         image:
-            card.image === undefined
+            serializedMediaFragment === undefined
                 ? undefined
-                : Image.fromBase64(
+                : MediaFragment.fromBase64(
                       source,
                       card.subtitle.start,
-                      card.image.base64,
-                      card.image.extension,
-                      card.image.error
+                      serializedMediaFragment.base64,
+                      serializedMediaFragment.extension,
+                      serializedMediaFragment.error
                   ),
         word: card.word,
         source: source,
@@ -476,40 +505,27 @@ export class Anki {
         const gui = mode === 'gui';
         const updateLast = mode === 'updateLast';
 
-        if (this.settingsProvider.audioField && audioClip && audioClip.error === undefined) {
-            const sanitizedName = this._sanitizeFileName(audioClip.name);
-            const data = await audioClip.base64();
-
-            if (data) {
-                if (gui || updateLast) {
-                    const fileName = (await this._storeMediaFile(sanitizedName, data, ankiConnectUrl)).result;
-                    this._appendField(fields, this.settingsProvider.audioField, `[sound:${fileName}]`, false);
-                } else {
-                    params.note['audio'] = {
-                        filename: sanitizedName,
-                        data,
-                        fields: [this.settingsProvider.audioField],
-                    };
-                }
-            }
+        const recentNotes = updateLast ? await this.findNotes('added:1', ankiConnectUrl) : [];
+        if (updateLast && recentNotes.length === 0) {
+            throw new Error('Could not find note to update');
         }
 
-        if (this.settingsProvider.imageField && image && image.error === undefined) {
-            const sanitizedName = this._sanitizeFileName(image.name);
-            const data = await image.base64();
-
-            if (data) {
-                if (gui || updateLast) {
-                    const fileName = (await this._storeMediaFile(sanitizedName, data, ankiConnectUrl)).result;
-                    this._appendField(fields, this.settingsProvider.imageField, `<img src="${fileName}">`, false);
-                } else {
-                    params.note['picture'] = {
-                        filename: sanitizedName,
-                        data,
-                        fields: [this.settingsProvider.imageField],
-                    };
-                }
-            }
+        const exportableAudio =
+            this.settingsProvider.audioField && audioClip && audioClip.error === undefined ? audioClip : undefined;
+        const exportableImage =
+            this.settingsProvider.imageField && image && image.error === undefined ? image : undefined;
+
+        const [encodedAudio, encodedImage] = await Promise.all([
+            this._encodeMedia(exportableAudio, 'audio'),
+            this._encodeMedia(exportableImage, image?.extension === 'webm' ? 'clip' : 'image'),
+        ]);
+
+        if (encodedAudio) {
+            await this._attachAudio(params, fields, encodedAudio, gui || updateLast, ankiConnectUrl);
+        }
+
+        if (encodedImage && image) {
+            await this._attachMediaFragment(params, fields, encodedImage, image, gui || updateLast, ankiConnectUrl);
         }
 
         params.note['fields'] = fields;
@@ -518,17 +534,9 @@ export class Anki {
             case 'gui':
                 return (await this._executeAction('guiAddCards', params, ankiConnectUrl)).result;
             case 'updateLast':
-                const recentNotes = (
-                    await this._executeAction('findNotes', { query: 'added:1' }, ankiConnectUrl)
-                ).result.sort();
-
-                if (recentNotes.length === 0) {
-                    throw new Error('Could not find note to update');
-                }
-
-                const lastNoteId = recentNotes[recentNotes.length - 1];
+                const lastNoteId = [...recentNotes].sort()[recentNotes.length - 1];
                 params.note['id'] = lastNoteId;
-                const infoResponse = await this._executeAction('notesInfo', { notes: [lastNoteId] });
+                const infoResponse = await this._executeAction('notesInfo', { notes: [lastNoteId] }, ankiConnectUrl);
 
                 if (infoResponse.result.length > 0 && infoResponse.result[0].noteId === lastNoteId) {
                     const info = infoResponse.result[0];
@@ -587,6 +595,92 @@ export class Anki {
         fields[fieldName] = newValue;
     }
 
+    private async _encodeMedia(media: Base64Exportable | undefined, type: string): Promise<EncodedMedia | undefined> {
+        if (!media) {
+            return undefined;
+        }
+
+        const sanitizedName = this._sanitizeFileName(media.name);
+        const data = await timedMediaBase64(type, media.extension, sanitizedName, () => media.base64());
+
+        if (!data) {
+            return undefined;
+        }
+
+        return { sanitizedName, data };
+    }
+
+    private async _attachAudio(
+        params: any,
+        fields: any,
+        encodedAudio: EncodedMedia,
+        storeMediaFile: boolean,
+        ankiConnectUrl?: string
+    ) {
+        if (storeMediaFile) {
+            await this._storeAndAppendField(
+                fields,
+                this.settingsProvider.audioField,
+                encodedAudio,
+                (fileName) => `[sound:${fileName}]`,
+                ankiConnectUrl
+            );
+            return;
+        }
+
+        params.note['audio'] = {
+            filename: encodedAudio.sanitizedName,
+            data: encodedAudio.data,
+            fields: [this.settingsProvider.audioField],
+        };
+    }
+
+    private async _attachMediaFragment(
+        params: any,
+        fields: any,
+        encodedImage: EncodedMedia,
+        image: MediaFragment,
+        storeMediaFile: boolean,
+        ankiConnectUrl?: string
+    ) {
+        if (image.extension === 'webm' || storeMediaFile) {
+            await this._storeAndAppendField(
+                fields,
+                this.settingsProvider.imageField,
+                encodedImage,
+                (fileName) => this._mediaFragmentFieldHtml(fileName, image.extension),
+                ankiConnectUrl
+            );
+            return;
+        }
+
+        params.note['picture'] = {
+            filename: encodedImage.sanitizedName,
+            data: encodedImage.data,
+            fields: [this.settingsProvider.imageField],
+        };
+    }
+
+    private async _storeAndAppendField(
+        fields: any,
+        fieldName: string | undefined,
+        encodedMedia: EncodedMedia,
+        value: (fileName: string) => string,
+        ankiConnectUrl?: string
+    ) {
+        const fileName = (await this._storeMediaFile(encodedMedia.sanitizedName, encodedMedia.data, ankiConnectUrl))
+            .result;
+        this._appendField(fields, fieldName, value(fileName), false);
+    }
+
+    private _mediaFragmentFieldHtml(fileName: string, extension: string) {
+        if (extension === 'webm') {
+            return `<video autoplay loop muted playsinline src="${fileName}"></video>`;
+        }
+
+        return `<img src="${fileName}">`;
+    }
+
     private _sanitizeUnsafeURLChars(name: string) {
         return name.replace(unsafeURLChars, replacement);
     }

common/app/components/App.tsx

@@ -4,7 +4,7 @@ import { type Theme } from '@mui/material/styles';
 import ThemeProvider from '@mui/material/styles/ThemeProvider';
 import { useWindowSize } from '../hooks/use-window-size';
 import {
-    Image,
+    MediaFragment,
     SubtitleModel,
     VideoTabModel,
     LegacyPlayerSyncMessage,
@@ -19,7 +19,7 @@ import {
     DownloadImageMessage,
     DownloadAudioMessage,
     CardTextFieldValues,
-    ImageErrorCode,
+    MediaFragmentErrorCode,
     RequestSubtitlesResponse,
 } from '@project/common';
 import { createTheme } from '@project/common/theme';
@@ -506,10 +506,14 @@ function App({
                         track3: extractText(card.subtitle, card.surroundingSubtitles, 2),
                         definition: newCard.definition ?? '',
                         audioClip: audioClip,
-                        image: Image.fromCard(
+                        image: MediaFragment.fromCard(
                             newCard,
                             settingsRef.current.maxImageWidth,
-                            settingsRef.current.maxImageHeight
+                            settingsRef.current.maxImageHeight,
+                            settingsRef.current.mediaFragmentFormat,
+                            settingsRef.current.mediaFragmentTrimStart,
+                            settingsRef.current.mediaFragmentTrimEnd,
+                            settingsRef.current.mediaFragmentMaxClipLength
                         ),
                         word: newCard.word ?? '',
                         source: `${newCard.subtitleFileName} (${humanReadableTime(card.mediaTimestamp)})`,
@@ -626,20 +630,37 @@ function App({
     const handleDownloadImage = useCallback(
         (item: CardModel) => {
             try {
-                const image = Image.fromCard(item, settings.maxImageWidth, settings.maxImageHeight)!;
+                const image = MediaFragment.fromCard(
+                    item,
+                    settings.maxImageWidth,
+                    settings.maxImageHeight,
+                    settings.mediaFragmentFormat,
+                    settings.mediaFragmentTrimStart,
+                    settings.mediaFragmentTrimEnd,
+                    settings.mediaFragmentMaxClipLength
+                )!;
 
                 if (image.error === undefined) {
                     image.download();
-                } else if (image.error === ImageErrorCode.fileLinkLost) {
+                } else if (image.error === MediaFragmentErrorCode.fileLinkLost) {
                     handleError(t('ankiDialog.imageFileLinkLost'));
-                } else if (image.error === ImageErrorCode.captureFailed) {
+                } else if (image.error === MediaFragmentErrorCode.captureFailed) {
                     handleError(t('ankiDialog.imageCaptureFailed'));
                 }
             } catch (e) {
                 handleError(e);
             }
         },
-        [handleError, settings.maxImageWidth, settings.maxImageHeight, t]
+        [
+            handleError,
+            settings.maxImageWidth,
+            settings.maxImageHeight,
+            settings.mediaFragmentFormat,
+            settings.mediaFragmentTrimStart,
+            settings.mediaFragmentTrimEnd,
+            settings.mediaFragmentMaxClipLength,
+            t,
+        ]
     );
 
     const handleDownloadCopyHistorySectionAsSrt = useCallback(

common/app/components/CopyHistoryList.tsx

@@ -22,7 +22,7 @@ import Typography from '@mui/material/Typography';
 import { type Theme } from '@mui/material';
 import { CopyHistoryItem } from '../..';
 import { AudioClip } from '../../audio-clip';
-import { Image } from '../..';
+import { MediaFragment } from '../..';
 
 interface CopyHistoryListProps {
     open: boolean;
@@ -107,7 +107,7 @@ const useImageAvailability = (item: CopyHistoryItem) => {
     const [isImageAvailable, setIsImageAvailable] = useState<boolean>();
 
     useEffect(() => {
-        const image = Image.fromCard(item, 0, 0);
+        const image = MediaFragment.fromCard(item, 0, 0);
 
         if (image) {
             setIsImageAvailable(image.error === undefined);

common/app/hooks/use-anki.ts

@@ -79,6 +79,18 @@ class SettingsAccessor {
     get maxImageHeight() {
         return this.settings.maxImageHeight;
     }
+    get mediaFragmentFormat() {
+        return this.settings.mediaFragmentFormat;
+    }
+    get mediaFragmentTrimStart() {
+        return this.settings.mediaFragmentTrimStart;
+    }
+    get mediaFragmentTrimEnd() {
+        return this.settings.mediaFragmentTrimEnd;
+    }
+    get mediaFragmentMaxClipLength() {
+        return this.settings.mediaFragmentMaxClipLength;
+    }
     get surroundingSubtitlesCountRadius() {
         return this.settings.surroundingSubtitlesCountRadius;
     }

common/app/services/video-channel.ts

@@ -589,6 +589,10 @@ export default class VideoChannel {
             audioPaddingEnd,
             maxImageWidth,
             maxImageHeight,
+            mediaFragmentFormat,
+            mediaFragmentTrimStart,
+            mediaFragmentTrimEnd,
+            mediaFragmentMaxClipLength,
             surroundingSubtitlesCountRadius,
             surroundingSubtitlesTimeRadius,
             ankiFieldSettings,
@@ -618,6 +622,10 @@ export default class VideoChannel {
                 audioPaddingEnd,
                 maxImageWidth,
                 maxImageHeight,
+                mediaFragmentFormat,
+                mediaFragmentTrimStart,
+                mediaFragmentTrimEnd,
+                mediaFragmentMaxClipLength,
                 surroundingSubtitlesCountRadius,
                 surroundingSubtitlesTimeRadius,
                 ankiFieldSettings,