#70 - Improve Timestamp Accuracy

cleansio/speech/timestamp.py

@@ -1,6 +1,6 @@
 """ Locates where words are located in an audio chunk """
 
-from utils import gcs_time_to_ms
+from utils import gcs_time_to_ms, num_syllables
 
 class Timestamp():
     """ Words are located by either assessing silence or by estimatingself.
@@ -20,7 +20,8 @@ def __compute_timestamps(self):
         """ Goes through each word in the chunk and computes the timestamps """
         if not self.lyrics.results:
             return None
-        return self.__parse_timestamps()
+        timestamps = self.__parse_timestamps()
+        return self.__improve_timestamps(timestamps)
 
     def __parse_timestamps(self):
         """ Parses GCS's output and returns [{word:str, start:ms, end:ms},].
@@ -33,3 +34,44 @@ def __parse_timestamps(self):
                 'end': gcs_time_to_ms(word_dict.end_time)
             })
         return timestamps
+
+    def __improve_timestamps(self, timestamps):
+        """ Adjusts the GCS's timestamps to be more accurate. O(n) """
+        num_stamps = len(timestamps)
+        for i in range(num_stamps):
+            accuracy = self.__timestamp_accuracy(timestamps[i])
+            # Too long, shorten one end
+            if accuracy == 1 and i > 1:
+                timestamps[i] = self.__shorten_timestamp(
+                    timestamps[i], timestamps[i - 1])
+            # Too short, lengthen one end
+            elif accuracy == -1 and i < num_stamps - 1:
+                timestamps[i] = self.__lengthen_timestamp(
+                    timestamps[i], timestamps[i + 1])
+        return timestamps
+
+    def __shorten_timestamp(self, timestamp, past):
+        """ Shorten the length of a timestamp """
+        past_accuracy = self.__timestamp_accuracy(past)
+        # Trim more from the beginning if past is short
+        timestamp['start'] += 200 if past_accuracy == -1 else 100
+        return timestamp
+
+    def __lengthen_timestamp(self, timestamp, future):
+        """ Lengthen the length of a timestamp """
+        future_accuracy = self.__timestamp_accuracy(future)
+        # Add more to the end if future is long
+        timestamp['end'] += 200 if future_accuracy == 1 else 100
+        return timestamp
+
+    @classmethod
+    def __timestamp_accuracy(cls, timestamp):
+        """ -1: Too short, 0: Accurate, 1: Too long """
+        syllables = num_syllables(timestamp['word'])
+        # On average a human says 1 syllable every 200 ms
+        expected_syll = (timestamp['end'] - timestamp['start']) / 200
+        if expected_syll - 2 > syllables:   # Too long
+            return 1
+        elif expected_syll + 2 < syllables: # Too short
+            return -1
+        return 0

cleansio/utils/__init__.py

@@ -4,4 +4,5 @@
 from .cli import setup_cli_args
 from .env import create_env_var
 from .files import create_temp_dir, file_name_no_ext
+from .language import num_syllables
 from .numbers import gcs_time_to_ms, is_number, leading_zero

cleansio/utils/language.py

@@ -0,0 +1,8 @@
+""" Utillity functions for language related work """
+
+import re
+
+def num_syllables(word):
+    """ Counts the number of syllables in a word """
+    # https://codegolf.stackexchange.com/a/47325
+    return len(re.findall(r'[aiouy]+e*|e(?!d$|ly).|[td]ed|le$', word))