vfp: support pitch and audio track generation

Author: SoulMelody

libresvip/plugins/ustx/ustx_parser.py

@@ -168,7 +168,7 @@ def parse_notes(self, notes: list[UNote], tick_prefix: int) -> list[Note]:
                     note.pronunciation = to_romaji(ustx_note.lyric)
                 elif (chinese_char := CHINESE_RE.search(ustx_note.lyric)) is not None:
                     note.pronunciation = " ".join(pypinyin.lazy_pinyin(chinese_char.group()))
-                else:
+                elif not ustx_note.lyric.startswith("+"):
                     note.pronunciation = ustx_note.lyric.removeprefix("?")
             if prev_ustx_note is not None:
                 if prev_ustx_note.end > ustx_note.position:

libresvip/plugins/vfp/model.py

@@ -42,8 +42,8 @@ class VOXFactoryClipBase(BaseModel):
     audio_data_key: Optional[str] = Field(None, alias="audioDataKey")
     audio_data_order: list[str] = Field(default_factory=list, alias="audioDataOrder")
     audio_data_quarter: float = Field(0, alias="audioDataQuarter")
-    note_bank: dict[str, VOXFactoryNote] = Field(alias="noteBank")
-    note_order: list[str] = Field(alias="noteOrder")
+    note_bank: dict[str, VOXFactoryNote] = Field(default_factory=dict, alias="noteBank")
+    note_order: list[str] = Field(default_factory=list, alias="noteOrder")
     next_note_index: int = Field(0, alias="nextNoteIndex")
     pinned_audio_data_order: list[str] = Field(default_factory=list, alias="pinnedAudioDataOrder")
     metadata: Optional[VOXFactoryMetadata] = None

libresvip/plugins/vfp/vox_factory_generator.py

@@ -1,13 +1,32 @@
 import dataclasses
+import functools
 import math
 import pathlib
 import secrets
 
-from libresvip.core.constants import DEFAULT_BPM, TICKS_IN_BEAT
+import more_itertools
+import portion
+
+from libresvip.core.constants import DEFAULT_BPM, DEFAULT_PHONEME, TICKS_IN_BEAT
+from libresvip.core.time_interval import PiecewiseIntervalDict
 from libresvip.core.time_sync import TimeSynchronizer
-from libresvip.model.base import Note, Project, SingingTrack, SongTempo, TimeSignature, Track
+from libresvip.model.base import (
+    InstrumentalTrack,
+    Note,
+    ParamCurve,
+    Params,
+    Project,
+    SongTempo,
+    TimeSignature,
+    Track,
+)
+from libresvip.utils.audio import audio_track_info
+from libresvip.utils.music_math import linear_interpolation
 
 from .model import (
+    VOXFactoryAudioClip,
+    VOXFactoryAudioData,
+    VOXFactoryAudioTrack,
     VOXFactoryNote,
     VOXFactoryProject,
     VOXFactoryTrack,
@@ -23,17 +42,20 @@ class VOXFactoryGenerator:
     prefix: str = dataclasses.field(init=False)
     audio_paths: dict[str, pathlib.Path] = dataclasses.field(default_factory=dict)
     synchronizer: TimeSynchronizer = dataclasses.field(init=False)
+    first_bar_length: int = dataclasses.field(init=False)
 
     def generate_project(self, project: Project) -> VOXFactoryProject:
         self.prefix = secrets.token_hex(5)
+        self.first_bar_length = int(project.time_signature_list[0].bar_length())
         self.synchronizer = TimeSynchronizer(project.song_tempo_list)
-        vox_project = VOXFactoryProject(
+        track_bank, audio_data_bank = self.generate_tracks(project.track_list)
+        return VOXFactoryProject(
             tempo=self.generate_tempo(project.song_tempo_list),
             time_signature=self.generate_time_signature(project.time_signature_list),
-            track_bank=self.generate_tracks(project.track_list),
+            track_bank=track_bank,
+            track_order=sorted(track_bank.keys()),
+            audio_data_bank=audio_data_bank,
         )
-        vox_project.track_order = sorted(vox_project.track_bank.keys())
-        return vox_project
 
     def generate_tempo(self, tempos: list[SongTempo]) -> float:
         return tempos[0].bpm if tempos else DEFAULT_BPM
@@ -44,25 +66,60 @@ def generate_time_signature(self, time_signatures: list[TimeSignature]) -> list[
         else:
             return [4, 4]
 
-    def generate_tracks(self, tracks: list[Track]) -> dict[str, VOXFactoryTrack]:
+    def generate_tracks(
+        self, tracks: list[Track]
+    ) -> tuple[dict[str, VOXFactoryTrack], dict[str, VOXFactoryAudioData]]:
         track_bank = {}
+        audio_data_bank = {}
         for i, track in enumerate(tracks):
-            if isinstance(track, SingingTrack):
-                clip_bank = self.generate_notes(track.note_list)
+            if isinstance(track, InstrumentalTrack):
+                audio_path = pathlib.Path(track.audio_file_path)
+                if (track_info := audio_track_info(track.audio_file_path)) is not None:
+                    source_audio_data_key = f"{self.prefix}-au{i}{audio_path.suffix}"
+                    self.audio_paths[source_audio_data_key] = audio_path
+                    audio_data_bank[source_audio_data_key] = VOXFactoryAudioData(
+                        sample_rate=track_info.sampling_rate,
+                        sample_length=int(track_info.duration * track_info.sampling_rate / 1000),
+                        number_of_channels=track_info.channel_s,
+                    )
+                    clip_bank = {
+                        f"{self.prefix}-cl0": VOXFactoryAudioClip(
+                            name=audio_path.stem,
+                            offset_quarter=0,
+                            start_quarter=track.offset / TICKS_IN_BEAT,
+                            length=track_info.duration / 1000,
+                            source_audio_data_key=source_audio_data_key,
+                        )
+                    }
+                    clip_order = [f"{self.prefix}-cl0"]
+                    track_bank[f"{self.prefix}-tr{i}"] = VOXFactoryAudioTrack(
+                        clip_bank=clip_bank,
+                        clip_order=clip_order,
+                        name=track.title,
+                        mute=track.mute,
+                        solo=track.solo,
+                        pan=track.pan,
+                    )
+            else:
+                clip_bank = self.generate_notes(track.note_list, track.edited_params)
                 clip_order = sorted(clip_bank.keys())
                 track_bank[f"{self.prefix}-tr{i}"] = VOXFactoryVocalTrack(
                     clip_bank=clip_bank,
                     clip_order=clip_order,
+                    name=track.title,
+                    mute=track.mute,
+                    solo=track.solo,
+                    pan=track.pan,
                 )
-        return track_bank
+        return track_bank, audio_data_bank
 
-    def generate_notes(self, notes: list[Note]) -> dict[str, VOXFactoryVocalClip]:
+    def generate_notes(self, notes: list[Note], params: Params) -> dict[str, VOXFactoryVocalClip]:
         note_bank = {}
         note_order = []
         max_ticks = notes[-1].end_pos if notes else 0
         max_quarter = max_ticks / TICKS_IN_BEAT
         for i, note in enumerate(notes):
-            note_bank[f"{self.prefix}-no{i}"] = self.generate_note(note)
+            note_bank[f"{self.prefix}-no{i}"] = self.generate_note(note, params)
             note_order.append(f"{self.prefix}-no{i}")
         clip_count = math.ceil(max_quarter / 32)
         clip_bank = {}
@@ -76,13 +133,48 @@ def generate_notes(self, notes: list[Note]) -> dict[str, VOXFactoryVocalClip]:
             )
         return clip_bank
 
-    def generate_note(self, note: Note) -> VOXFactoryNote:
+    def generate_note(self, note: Note, params: Params) -> VOXFactoryNote:
         note_start_time = self.synchronizer.get_actual_secs_from_ticks(note.start_pos)
         return VOXFactoryNote(
             time=note_start_time,
             ticks=note.start_pos,
             duration_ticks=note.length,
             midi=note.key_number,
             name=note.lyric,
-            syllable=note.pronunciation,
+            syllable=note.pronunciation or DEFAULT_PHONEME,
+            pitch_bends=self.generate_note_pitch(note, params.pitch),
         )
+
+    def generate_note_pitch(self, note: Note, pitch: ParamCurve) -> list[float]:
+        note_start_time = self.synchronizer.get_actual_secs_from_ticks(note.start_pos)
+        note_end_time = self.synchronizer.get_actual_secs_from_ticks(note.end_pos)
+        key_interval_dict = PiecewiseIntervalDict()
+        secs_step = 1024 / 44100
+        prev_secs = None
+        prev_key: float = -1
+        for point in pitch.points.root:
+            if point.x - self.first_bar_length < note.start_pos:
+                continue
+            elif point.x - self.first_bar_length > note.end_pos:
+                break
+            if point.y == -100:
+                prev_secs = None
+                prev_key = 0
+            else:
+                secs = self.synchronizer.get_actual_secs_from_ticks(point.x - self.first_bar_length)
+                key = point.y / 100
+                if prev_secs is not None:
+                    key_interval_dict[portion.openclosed(prev_secs, secs)] = functools.partial(
+                        linear_interpolation,
+                        start=(prev_secs, prev_key - note.key_number),
+                        end=(secs, key - note.key_number),
+                    )
+                else:
+                    key_interval_dict[portion.singleton(secs)] = key - note.key_number
+                prev_secs = secs
+                prev_key = key
+        pitch_bends = [
+            key_interval_dict.get(secs, 0)
+            for secs in more_itertools.numeric_range(note_start_time, note_end_time, secs_step)
+        ]
+        return pitch_bends if any(pitch_bends) else []

libresvip/plugins/vpr/vpr_generator.py

@@ -164,7 +164,7 @@ def generate_tracks(self, track_list: list[Track]) -> list[VocaloidTracks]:
                             is_solo_mode=track.solo,
                         )
                     )
-            elif isinstance(track, SingingTrack):
+            else:
                 singing_track_found = True
                 notes = [
                     VocaloidNotes(