feat(subtitles): add Netflix streaming subtitle adapter

.changeset/streaming-netflix-adapter.md

@@ -0,0 +1,5 @@
+---
+"@read-frog/extension": patch
+---
+
+Add a minimal streaming subtitle adapter path with Netflix as the first adapter. Netflix switches the player to an English caption track in the page world, captures player-rendered caption cues, and feeds them through the existing Read Frog translation pipeline.

src/entrypoints/interceptor.content/index.ts

@@ -1,12 +1,15 @@
 import { defineContentScript } from "#imports"
 import { injectPlayerApi } from "./inject-player-api"
+import { injectStreamingSubtitlesInterceptor } from "./streaming-subtitles-interceptor"
 
 export default defineContentScript({
-  matches: ["*://*.youtube.com/*", "*://*.youtube-nocookie.com/*"],
+  matches: ["*://*.youtube.com/*", "*://*.youtube-nocookie.com/*", "*://*.netflix.com/*"],
   allFrames: true,
   world: "MAIN",
   runAt: "document_start",
   main() {
-    injectPlayerApi()
+    if (/(?:^|\.)youtube(?:-nocookie)?\.com$/i.test(window.location.hostname))
+      injectPlayerApi()
+    injectStreamingSubtitlesInterceptor()
   },
 })

src/entrypoints/interceptor.content/streaming-subtitles-interceptor.ts

@@ -0,0 +1,129 @@
+import {
+  STREAMING_ENSURE_NATIVE_SUBTITLES_TYPE,
+  STREAMING_LIVE_CAPTURE_CUE_TYPE,
+  STREAMING_LIVE_CAPTURE_START_TYPE,
+  STREAMING_LIVE_CAPTURE_STOP_TYPE,
+  STREAMING_SUBTITLE_TRACKS_TYPE,
+} from "@/utils/constants/subtitles"
+import { startDomLiveCapture } from "@/utils/subtitles/fetchers/streaming/live-capture"
+
+interface InterceptedTrack {
+  id?: string
+  language?: string
+  label?: string
+  kind?: string
+  pagePath?: string
+}
+
+declare global {
+  interface Window {
+    __READ_FROG_STREAMING_SUBTITLES_INTERCEPTOR__?: boolean
+  }
+}
+
+const NETFLIX_HOST = /(?:^|\.)netflix\.com$/i
+const NETFLIX_PLAYER_TRACK_KIND = "netflix-player"
+const NETFLIX_CAPTION_SELECTOR = ".player-timedtext-text-container, [data-uia='player-timedtext'], .player-timedtext"
+let stopNetflixLiveCapture: (() => void) | null = null
+
+function pagePath(): string {
+  return window.location.pathname
+}
+
+function trackKey(track: InterceptedTrack): string {
+  return track.id ?? track.label ?? track.language ?? ""
+}
+
+function postTracks(tracks: InterceptedTrack[]) {
+  const unique = [...new Map(tracks.map(track => [trackKey(track), track])).values()]
+  if (unique.length)
+    window.postMessage({ type: STREAMING_SUBTITLE_TRACKS_TYPE, tracks: unique }, window.location.origin)
+}
+
+function getNetflixPlayer(): any {
+  const videoPlayer = (window as any).netflix?.appContext?.state?.playerApp?.getAPI?.()?.videoPlayer
+  const sessionId = videoPlayer?.getAllPlayerSessionIds?.()[0]
+  return sessionId ? videoPlayer.getVideoPlayerBySessionId(sessionId) : null
+}
+
+function netflixPlayerTracks(): InterceptedTrack[] {
+  const tracks = getNetflixPlayer()?.getTimedTextTrackList?.() ?? []
+  return tracks
+    .filter((track: any) => !track.isNoneTrack)
+    .map((track: any) => ({
+      id: track.trackId,
+      language: track.bcp47,
+      label: track.displayName,
+      kind: `${NETFLIX_PLAYER_TRACK_KIND} ${track.rawTrackType ?? ""} ${track.trackType ?? ""}`,
+      pagePath: pagePath(),
+    }))
+}
+
+function getNetflixSourceTrack(): any {
+  const tracks = getNetflixPlayer()?.getTimedTextTrackList?.() ?? []
+  return tracks.find((track: any) => track.bcp47 === "en" && /caption|assistive/i.test(`${track.rawTrackType ?? ""} ${track.trackType ?? ""}`))
+    ?? tracks.find((track: any) => track.bcp47 === "en" && !track.isNoneTrack)
+    ?? tracks.find((track: any) => !track.isNoneTrack)
+}
+
+function readNativeCaptionText(): string {
+  const seen = new Set<string>()
+  return [...document.querySelectorAll(NETFLIX_CAPTION_SELECTOR)]
+    .map(node => node.textContent?.trim() ?? "")
+    .filter(text => text && !seen.has(text) && seen.add(text))
+    .join("\n")
+}
+
+function currentVideoTimeMs(): number {
+  const video = document.querySelector("video") as HTMLVideoElement | null
+  return Math.round((video?.currentTime ?? 0) * 1000)
+}
+
+function postLiveCue(fragments: Array<{ text: string, start: number, end: number }>) {
+  window.postMessage({ type: STREAMING_LIVE_CAPTURE_CUE_TYPE, fragments }, window.location.origin)
+}
+
+function startNetflixLiveCapture() {
+  const player = getNetflixPlayer()
+  const track = getNetflixSourceTrack()
+  if (!player || !track)
+    return null
+
+  const previous = player.getTimedTextTrack?.()
+  player.setTimedTextTrack(track)
+  const stop = startDomLiveCapture({
+    readText: readNativeCaptionText,
+    currentTimeMs: currentVideoTimeMs,
+    onFragments: postLiveCue,
+  })
+
+  return () => {
+    stop()
+    if (previous)
+      player.setTimedTextTrack(previous)
+  }
+}
+
+export function injectStreamingSubtitlesInterceptor() {
+  if (window.__READ_FROG_STREAMING_SUBTITLES_INTERCEPTOR__ || !NETFLIX_HOST.test(window.location.hostname))
+    return
+  window.__READ_FROG_STREAMING_SUBTITLES_INTERCEPTOR__ = true
+
+  window.addEventListener("message", (event) => {
+    if (event.origin !== window.location.origin)
+      return
+    if (event.data?.type === STREAMING_ENSURE_NATIVE_SUBTITLES_TYPE) {
+      postTracks(netflixPlayerTracks())
+      return
+    }
+    if (event.data?.type === STREAMING_LIVE_CAPTURE_START_TYPE) {
+      stopNetflixLiveCapture?.()
+      stopNetflixLiveCapture = startNetflixLiveCapture()
+      return
+    }
+    if (event.data?.type === STREAMING_LIVE_CAPTURE_STOP_TYPE) {
+      stopNetflixLiveCapture?.()
+      stopNetflixLiveCapture = null
+    }
+  })
+}

src/entrypoints/subtitles.content/index.tsx

@@ -8,26 +8,79 @@ declare global {
   }
 }
 
+const NETFLIX_WATCH_PATH_PATTERN = /^\/watch\//
+const URL_CHANGE_EVENT = "extension:URLChange"
+
+function isNetflixPage(): boolean {
+  return /(?:^|\.)netflix\.com$/i.test(window.location.hostname)
+}
+
+function isPlaybackReady(): boolean {
+  return NETFLIX_WATCH_PATH_PATTERN.test(window.location.pathname)
+}
+
+function watchStreamingUrlChanges(): () => void {
+  let previousUrl = window.location.href
+  // ponytail: SPA URL polling, replace with a shared history listener if latency matters.
+  const intervalId = setInterval(() => {
+    const currentUrl = window.location.href
+    if (currentUrl === previousUrl)
+      return
+    const from = previousUrl
+    previousUrl = currentUrl
+    window.dispatchEvent(new CustomEvent(URL_CHANGE_EVENT, { detail: { from, to: currentUrl, reason: "interval" } }))
+  }, 500)
+  return () => clearInterval(intervalId)
+}
+
 export default defineContentScript({
-  matches: ["*://*.youtube.com/*", "*://*.youtube-nocookie.com/*"],
+  matches: [
+    "*://*.youtube.com/*",
+    "*://*.youtube-nocookie.com/*",
+    "*://*.netflix.com/*",
+  ],
   allFrames: true,
   cssInjectionMode: "manifest",
   async main(ctx) {
     if (window.__READ_FROG_SUBTITLES_INJECTED__)
       return
-    window.__READ_FROG_SUBTITLES_INJECTED__ = true
 
     const config = await getLocalConfig()
     if (!config?.videoSubtitles?.enabled) {
       window.__READ_FROG_SUBTITLES_INJECTED__ = false
       return
     }
 
+    const cleanupHandlers: Array<() => void> = []
     ctx.onInvalidated(() => {
+      cleanupHandlers.forEach(cleanup => cleanup())
       window.__READ_FROG_SUBTITLES_INJECTED__ = false
     })
 
-    const { bootstrapSubtitlesRuntime } = await import("./runtime")
-    await bootstrapSubtitlesRuntime()
+    const bootstrapRuntime = async () => {
+      if (window.__READ_FROG_SUBTITLES_INJECTED__)
+        return
+      window.__READ_FROG_SUBTITLES_INJECTED__ = true
+      const { bootstrapSubtitlesRuntime } = await import("./runtime")
+      await bootstrapSubtitlesRuntime()
+    }
+
+    if (isNetflixPage()) {
+      cleanupHandlers.push(watchStreamingUrlChanges())
+      if (!isPlaybackReady()) {
+        const onNavigate = () => {
+          if (!isPlaybackReady())
+            return
+          window.removeEventListener(URL_CHANGE_EVENT, onNavigate)
+          void bootstrapRuntime()
+        }
+        window.addEventListener(URL_CHANGE_EVENT, onNavigate)
+        cleanupHandlers.push(() => window.removeEventListener(URL_CHANGE_EVENT, onNavigate))
+        window.__READ_FROG_SUBTITLES_INJECTED__ = false
+        return
+      }
+    }
+
+    await bootstrapRuntime()
   },
 })

src/entrypoints/subtitles.content/platforms/netflix/config.ts

@@ -0,0 +1,26 @@
+import type { PlatformConfig } from "@/entrypoints/subtitles.content/platforms"
+import { STREAMING_NATIVE_SUBTITLES_SELECTOR } from "@/utils/constants/subtitles"
+
+const URL_CHANGE_EVENT = "extension:URLChange"
+
+export function getNetflixConfig(): PlatformConfig {
+  return {
+    selectors: {
+      video: "video",
+      playerContainer: "body",
+      nativeSubtitles: STREAMING_NATIVE_SUBTITLES_SELECTOR,
+    },
+    events: {
+      navigateStart: URL_CHANGE_EVENT,
+      navigateFinish: URL_CHANGE_EVENT,
+    },
+    controls: {
+      findVideoContainer: () => document.querySelector<HTMLElement>("video")?.parentElement ?? document.body,
+      measureHeight: () => 0,
+      checkVisibility: () => true,
+    },
+    // Only /watch/<id> pages carry a video id; browse/details pages return null so
+    // they don't trigger a navigation reset that would drop the next title's session.
+    getVideoId: () => window.location.pathname.match(/^\/watch\/([^/?#]+)/)?.[1] ?? null,
+  }
+}

src/entrypoints/subtitles.content/platforms/streaming.ts

@@ -0,0 +1,33 @@
+import type { PlatformConfig } from "@/entrypoints/subtitles.content/platforms"
+import type { StreamingSiteAdapter } from "@/utils/subtitles/fetchers/streaming/streaming-fetcher"
+import { netflixSiteAdapter } from "@/utils/subtitles/fetchers/netflix"
+import { StreamingSubtitlesFetcher } from "@/utils/subtitles/fetchers/streaming/streaming-fetcher"
+import { UniversalVideoAdapter } from "../universal-adapter"
+import { getNetflixConfig } from "./netflix/config"
+
+export interface StreamingSite {
+  adapter: StreamingSiteAdapter
+  getConfig: () => PlatformConfig
+}
+
+const STREAMING_SITES: StreamingSite[] = [
+  {
+    adapter: netflixSiteAdapter,
+    getConfig: getNetflixConfig,
+  },
+]
+
+export function findStreamingSite(url: URL): StreamingSite | null {
+  return STREAMING_SITES.find(site => site.adapter.matches(url)) ?? null
+}
+
+export function createStreamingSubtitlesAdapter(site: StreamingSite): { config: PlatformConfig, adapter: UniversalVideoAdapter } {
+  const config = site.getConfig()
+  return {
+    config,
+    adapter: new UniversalVideoAdapter({
+      config,
+      subtitlesFetcher: new StreamingSubtitlesFetcher(site.adapter),
+    }),
+  }
+}

src/entrypoints/subtitles.content/runtime.ts

@@ -1,12 +1,43 @@
+import type { StreamingSite } from "./platforms/streaming"
+import type { UniversalVideoAdapter } from "./universal-adapter"
+import { TRANSLATE_BUTTON_CONTAINER_ID } from "@/utils/constants/subtitles"
 import { initYoutubeSubtitles } from "./init-youtube-subtitles"
+import { createStreamingSubtitlesAdapter, findStreamingSite } from "./platforms/streaming"
+import { mountSubtitlesUI } from "./renderer/mount-subtitles-ui"
+import { renderSubtitlesTranslateButton } from "./renderer/render-translate-button"
+
+const FLOATING_BUTTON_HOST_ID = "read-frog-streaming-subtitles-floating-button-host"
 
 let hasBootstrappedSubtitlesRuntime = false
 
-export function bootstrapSubtitlesRuntime() {
+export async function bootstrapSubtitlesRuntime() {
   if (hasBootstrappedSubtitlesRuntime) {
     return
   }
 
   hasBootstrappedSubtitlesRuntime = true
-  initYoutubeSubtitles()
+
+  const site = findStreamingSite(new URL(window.location.href))
+  if (site)
+    await initStreamingSite(site)
+  else
+    initYoutubeSubtitles()
+}
+
+async function initStreamingSite(site: StreamingSite) {
+  const { config, adapter } = createStreamingSubtitlesAdapter(site)
+  await mountSubtitlesUI({ adapter, config })
+  mountFloatingButton(adapter)
+  await adapter.initialize()
+}
+
+function mountFloatingButton(adapter: UniversalVideoAdapter) {
+  if (document.getElementById(FLOATING_BUTTON_HOST_ID) || document.getElementById(TRANSLATE_BUTTON_CONTAINER_ID))
+    return
+
+  const host = document.createElement("div")
+  host.id = FLOATING_BUTTON_HOST_ID
+  host.style.cssText = "position:fixed;right:20px;bottom:96px;width:48px;height:48px;z-index:2147483647;pointer-events:auto;"
+  host.appendChild(renderSubtitlesTranslateButton({ adapter }))
+  document.body.appendChild(host)
 }

src/entrypoints/subtitles.content/subtitles-scheduler.ts

@@ -36,23 +36,26 @@ export class SubtitlesScheduler {
     for (const newSub of subtitles) {
       const existing = existingMap.get(newSub.start)
 
-      if (!existing) {
-        this.subtitles.push(newSub)
-        continue
-      }
-
-      if (newSub.translation) {
-        const updatedSub = { ...existing, translation: newSub.translation }
-        const idx = this.subtitles.findIndex(s => s.start === existing.start)
-        if (idx >= 0) {
-          this.subtitles[idx] = updatedSub
-        }
-
-        if (currentSubtitle && existing.start === currentSubtitle.start) {
-          currentSubtitleUpdated = true
-        }
-      }
-    }
+      if (!existing) {
+        this.subtitles.push(newSub)
+        continue
+      }
+
+      const updatedSub = {
+        ...existing,
+        text: newSub.text || existing.text,
+        end: newSub.end || existing.end,
+        translation: newSub.translation ?? existing.translation,
+      }
+      const idx = this.subtitles.findIndex(s => s.start === existing.start)
+      if (idx >= 0) {
+        this.subtitles[idx] = updatedSub
+      }
+
+      if (currentSubtitle && existing.start === currentSubtitle.start) {
+        currentSubtitleUpdated = true
+      }
+    }
 
     this.subtitles.sort((a, b) => a.start - b.start)
     this.updateSubtitles(this.videoElement.currentTime)