feat(synced-lyrics): romanization (#2790)

* feat(synced-lyrics): init romanization!

* remove debug logs and add TODO

* feat(synced-lyrics/romanization): Mandarin!

* feat(synced-lyrics/romanization): improve japanese detection

* feat(synced-lyrics/romanization): Korean!

* qol(synced-lyrics/romanization): canonicalize punctuation and symbols

* feat(synced-lyrics/romanization): handle japanese+korean and korean+chinese lyrics

* revert formatting on electron.vite.config.mts

* feat(synced-lyrics/romanization): romanize plain lyrics

* apply fix by @kimjammer

* fix lockfile due to rebase

* feat(synced-lyrics): improve lyric processing and formatting;

* feat(synced-lyrics/romanization): add option to enable/disable romanization

* chore: move default value for --lyrics-duration to the declaration

* update lockfile

* fix: improvement

1. improved language detection logic
2. changed code to work in the renderer process

* fix: fix regression (canonicalize)

---------

Co-authored-by: JellyBrick <shlee1503@naver.com>

src/plugins/synced-lyrics/renderer/components/LyricsContainer.tsx

@@ -1,4 +1,4 @@
-import { createSignal, For, Match, Show, Switch } from 'solid-js';
+import { createEffect, createSignal, For, Match, Show, Switch } from 'solid-js';
 
 import { SyncedLine } from './SyncedLine';
 
@@ -6,6 +6,7 @@ import { ErrorDisplay } from './ErrorDisplay';
 import { LoadingKaomoji } from './LoadingKaomoji';
 import { PlainLyrics } from './PlainLyrics';
 
+import { hasJapaneseInString, hasKoreanInString } from '../utils';
 import { currentLyrics, lyricsStore } from '../../providers';
 
 export const [debugInfo, setDebugInfo] = createSignal<string>();
@@ -13,6 +14,20 @@ export const [currentTime, setCurrentTime] = createSignal<number>(-1);
 
 // prettier-ignore
 export const LyricsContainer = () => {
+  const [hasJapanese, setHasJapanese] = createSignal<boolean>(false);
+  const [hasKorean, setHasKorean] = createSignal<boolean>(false);
+
+  createEffect(() => {
+    const data = currentLyrics()?.data;
+    if (data) {
+      setHasKorean(hasKoreanInString(data));
+      setHasJapanese(hasJapaneseInString(data));
+    } else {
+      setHasKorean(false);
+      setHasJapanese(false);
+    }
+  });
+
   return (
     <div class="lyric-container">
       <Switch>
@@ -42,12 +57,12 @@ export const LyricsContainer = () => {
       <Switch>
         <Match when={currentLyrics().data?.lines}>
           <For each={currentLyrics().data?.lines}>
-            {(item) => <SyncedLine line={item} />}
+            {(item) => <SyncedLine line={item} hasJapanese={hasJapanese()} hasKorean={hasKorean()} />}
           </For>
         </Match>
 
         <Match when={currentLyrics().data?.lyrics}>
-          <PlainLyrics lyrics={currentLyrics().data?.lyrics!} />
+          <PlainLyrics lyrics={currentLyrics().data!.lyrics!} hasJapanese={hasJapanese()} hasKorean={hasKorean()} />
         </Match>
       </Switch>
     </div>

src/plugins/synced-lyrics/renderer/components/PlainLyrics.tsx

@@ -1,28 +1,91 @@
-import { createMemo, For } from 'solid-js';
+import { createEffect, createMemo, createSignal, For, Show } from 'solid-js';
+
+import {
+  canonicalize,
+  romanizeChinese,
+  romanizeHangul,
+  romanizeJapanese,
+  romanizeJapaneseOrHangul,
+  simplifyUnicode,
+} from '../utils';
+import { config } from '../renderer';
 
 interface PlainLyricsProps {
   lyrics: string;
+  hasJapanese: boolean;
+  hasKorean: boolean;
 }
 
 export const PlainLyrics = (props: PlainLyricsProps) => {
-  const lines = createMemo(() => props.lyrics.split('\n'));
+  const lines = props.lyrics.split('\n').filter((line) => line.trim());
+  const [romanizedLines, setRomanizedLines] = createSignal<
+    Record<string, string>
+  >({});
+
+  const combinedLines = createMemo(() => {
+    const out = [];
+
+    for (let i = 0; i < lines.length; i++) {
+      out.push([lines[i], romanizedLines()[i]]);
+    }
+
+    return out;
+  });
+
+  createEffect(async () => {
+    if (!config()?.romanization) return;
+
+    for (let i = 0; i < lines.length; i++) {
+      let romanized: string;
+
+      if (props.hasJapanese) {
+        if (props.hasKorean)
+          romanized = await romanizeJapaneseOrHangul(lines[i]);
+        else romanized = await romanizeJapanese(lines[i]);
+      } else if (props.hasKorean) romanized = romanizeHangul(lines[i]);
+      else romanized = romanizeChinese(lines[i]);
+
+      setRomanizedLines((prev) => ({
+        ...prev,
+        [i]: canonicalize(romanized),
+      }));
+    }
+  });
 
   return (
     <div class="plain-lyrics">
-      <For each={lines()}>
-        {(line) => {
-          if (line.trim() === '') {
-            return <br />;
-          } else {
-            return (
+      <For each={combinedLines()}>
+        {([line, romanized]) => {
+          return (
+            <div
+              class={`${
+                line.match(/^\[.+\]$/s) ? 'lrc-header' : ''
+              } text-lyrics description ytmusic-description-shelf-renderer`}
+              style={{
+                'display': 'flex',
+                'flex-direction': 'column',
+              }}
+            >
               <yt-formatted-string
-                class="text-lyrics description ytmusic-description-shelf-renderer"
                 text={{
                   runs: [{ text: line }],
                 }}
               />
-            );
-          }
+              <Show
+                when={
+                  config()?.romanization &&
+                  simplifyUnicode(line) !== simplifyUnicode(romanized)
+                }
+              >
+                <yt-formatted-string
+                  class="romaji"
+                  text={{
+                    runs: [{ text: romanized }],
+                  }}
+                />
+              </Show>
+            </div>
+          );
         }}
       </For>
     </div>

src/plugins/synced-lyrics/renderer/components/SyncedLine.tsx

@@ -1,22 +1,33 @@
-import { createEffect, createMemo, For } from 'solid-js';
+import { createEffect, createMemo, For, Show, createSignal } from 'solid-js';
 
 import { currentTime } from './LyricsContainer';
 
 import { config } from '../renderer';
 import { _ytAPI } from '..';
 
+import {
+  canonicalize,
+  romanizeChinese,
+  romanizeHangul,
+  romanizeJapanese,
+  romanizeJapaneseOrHangul,
+  simplifyUnicode,
+} from '../utils';
+
 import type { LineLyrics } from '../../types';
 
 interface SyncedLineProps {
   line: LineLyrics;
+  hasJapanese: boolean;
+  hasKorean: boolean;
 }
 
-export const SyncedLine = ({ line }: SyncedLineProps) => {
+export const SyncedLine = (props: SyncedLineProps) => {
   const status = createMemo(() => {
     const current = currentTime();
 
-    if (line.timeInMs >= current) return 'upcoming';
-    if (current - line.timeInMs >= line.duration) return 'previous';
+    if (props.line.timeInMs >= current) return 'upcoming';
+    if (current - props.line.timeInMs >= props.line.duration) return 'previous';
     return 'current';
   });
 
@@ -28,8 +39,28 @@ export const SyncedLine = ({ line }: SyncedLineProps) => {
   });
 
   const text = createMemo(() => {
-    if (line.text.trim()) return line.text;
-    return config()?.defaultTextString ?? '';
+    if (!props.line.text.trim()) {
+      return config()?.defaultTextString ?? '';
+    }
+
+    return props.line.text;
+  });
+
+  const [romanization, setRomanization] = createSignal('');
+
+  createEffect(async () => {
+    if (!config()?.romanization) return;
+
+    const input = canonicalize(text());
+
+    let result: string;
+    if (props.hasJapanese) {
+      if (props.hasKorean) result = await romanizeJapaneseOrHangul(input);
+      else result = await romanizeJapanese(input);
+    } else if (props.hasKorean) result = romanizeHangul(input);
+    else result = romanizeChinese(input);
+
+    setRomanization(canonicalize(result));
   });
 
   if (!text()) {
@@ -47,35 +78,79 @@ export const SyncedLine = ({ line }: SyncedLineProps) => {
       ref={ref}
       class={`synced-line ${status()}`}
       onClick={() => {
-        _ytAPI?.seekTo(line.timeInMs / 1000);
+        _ytAPI?.seekTo(props.line.timeInMs / 1000);
       }}
     >
-      <div dir="auto" class="text-lyrics description ytmusic-description-shelf-renderer">
+      <div dir="auto" class="description ytmusic-description-shelf-renderer">
         <yt-formatted-string
           text={{
-            runs: [{ text: config()?.showTimeCodes ? `[${line.time}] ` : '' }],
+            runs: [
+              { text: config()?.showTimeCodes ? `[${props.line.time}] ` : '' },
+            ],
           }}
         />
 
-        <For each={text().split(' ')}>
-          {(word, index) => {
-            return (
-              <span
-                style={{
-                  'transition-delay': `${index() * 0.05}s`,
-                  'animation-delay': `${index() * 0.05}s`,
-                  '--lyrics-duration:': `${line.duration / 1000}s;`,
-                }}
-              >
-                <yt-formatted-string
-                  text={{
-                    runs: [{ text: `${word} ` }],
-                  }}
-                />
-              </span>
+        <div
+          class="text-lyrics"
+          ref={(div: HTMLDivElement) => {
+            // TODO: Investigate the animation, even though the duration is properly set, all lines have the same animation duration
+            div.style.setProperty(
+              '--lyrics-duration',
+              `${props.line.duration / 1000}s`,
+              'important',
             );
           }}
-        </For>
+          style={{ 'display': 'flex', 'flex-direction': 'column' }}
+        >
+          <span>
+            <For each={text().split(' ')}>
+              {(word, index) => {
+                return (
+                  <span
+                    style={{
+                      'transition-delay': `${index() * 0.05}s`,
+                      'animation-delay': `${index() * 0.05}s`,
+                    }}
+                  >
+                    <yt-formatted-string
+                      text={{
+                        runs: [{ text: `${word} ` }],
+                      }}
+                    />
+                  </span>
+                );
+              }}
+            </For>
+          </span>
+
+          <Show
+            when={
+              config()?.romanization &&
+              simplifyUnicode(text()) !== simplifyUnicode(romanization())
+            }
+          >
+            <span class="romaji">
+              <For each={romanization().split(' ')}>
+                {(word, index) => {
+                  return (
+                    <span
+                      style={{
+                        'transition-delay': `${index() * 0.05}s`,
+                        'animation-delay': `${index() * 0.05}s`,
+                      }}
+                    >
+                      <yt-formatted-string
+                        text={{
+                          runs: [{ text: `${word} ` }],
+                        }}
+                      />
+                    </span>
+                  );
+                }}
+              </For>
+            </span>
+          </Show>
+        </div>
       </div>
     </div>
   );