Merge pull request #566 from shorepine/pythonaudio

Support for AMY input and output buffer callbacks in Python

Author: bwhitman

amy

@@ -493,7 +493,7 @@ amy.send(voices='0', load_patch=101, note=50, vel=1) # all Alles speakers in a m
 amy.send(voices='0', load_patch=101, note=50, vel=1, client=2) # just a certain client
 ```
 
-To load your own WAVE files as samples, use `amy.load_sample`:
+To load your own WAVE files as samples you can play like an instrument, use `amy.load_sample`:
 
 ```python
 # To save space / RAM, you may want to downsample your WAVE files to 11025 or 22050Hz. We detect SR automatically.
@@ -543,6 +543,54 @@ for i,note in enumerate(chord.midinotes()):
     amy.send(wave=amy.SINE,osc=i*9,note=note,vel=0.25)
 ```
 
+## Low level sample access and direct audio playback
+
+You can access the audio input (`AUDIO_IN0/1`) sample buffer per frame (on Tulip Desktop, Web, and future devices with audio input support), and you can also set two external audio channels (`AUDIO_EXT0/1`) from Python. This lets you synthesize audio in Python, or do things like stream WAV files from disk to the audio output. 
+
+To do so, you need to register an AMY frame callback in Tulip. In this example, we open a WAV file and read it 256 frames per block, and set those frames to the EXT0/1 oscillators, which we initialize as AMY oscillators 0 and 1, with their pan set to left and right. 
+
+```python
+# Play a wav file through AMY, streaming from disk
+import amy_wave
+f = amy_wave.open(wav_filename,'rb')
+
+def cb(x):
+    frames = f.readframes(256)
+    if(len(frames)!=1024): # file done. stop the AMY frame callback.
+        frames = bytes(1024)
+        tulip.amy_block_done_callback()
+    # Sets the stereo channel buffer EXT0/EXT1 from the frames bytes
+    tulip.amy_set_external_input_buffer(frames)
+
+amy.reset()
+amy.send(osc=0,wave=amy.AUDIO_EXT0, pan=0, vel=1)
+amy.send(osc=1,wave=amy.AUDIO_EXT1, pan=1, vel=1)
+tulip.amy_block_done_callback(cb)
+```
+
+To sample incoming audio (on devices that support it), use `tulip.amy_get_input_buffer`:
+
+```python
+buf = bytes()
+tick_start = 0
+ms = 2000
+
+def sample(x):
+    global buf
+    buf = buf + tulip.amy_get_input_buffer()
+    # stop "recording" to buf after ms
+    if(tulip.ticks_ms() > tick_start + ms): 
+        tulip.amy_block_done_callback()
+        play()
+
+tick_start = tulip.ticks_ms()
+print("Recording for 2s. Make sure audio input is on!")
+tulip.amy_block_done_callback(sample)
+# Then buf will have stereo frames of audio to do whatever you want with.
+```
+
+Please note: on Tulip CC hardware, you do not have much compute time left per block to do much. Reading files, saving to memory or doing simple synthesis works, but most anything more complicated you should write your effects / synthesis code in C, as part of AMY.
+
 ## Music sequencer
 
 Tulip is always running AMY's live sequencer, which allows you to have multiple music programs running sharing a common clock.

docs/tulip_api.md

@@ -211,11 +211,11 @@ float cv_input_hook(uint16_t channel) {
     #endif
 }
 
-
+extern mp_obj_t audio_buffer_callback;
 // Write to the GP8413
 uint8_t cv_output_hook(uint16_t osc, SAMPLE * buf, uint16_t len) {
+    if(external_map[osc]==1 || external_map[osc]==2) {
 #ifdef ESP_PLATFORM
-    if(external_map[osc]>0) {
         // -5v to +5v? 
         float volts = S2F(buf[0])*5.0;
         int32_t val = (int32_t)(((volts + 10)/20.0) * 0x8000);
@@ -229,13 +229,16 @@ uint8_t cv_output_hook(uint16_t osc, SAMPLE * buf, uint16_t len) {
         uint8_t channel = external_map[osc]-1;
         if(channel == 1) ch = 0x04;
         bytes[0] = ch;
-        //fprintf(stderr, "writing %f volts [%ld] to channel %d\n", volts, val, channel);
         i2c_master_write_to_device(I2C_NUM_0, addr, bytes, 3, pdMS_TO_TICKS(10));
+        // silence this output
         return 1;
-    }
 #endif
+        return 0;
+    } else if(external_map[osc]>2) { // python audio buffer callback, WIP
+        
+        return 0;
+    } 
     return 0;
-
 }
 
 #ifdef ESP_PLATFORM
@@ -276,7 +279,9 @@ void amyboard_fill_audio_buffer_task() {
         if(written != AMY_BLOCK_SIZE * sizeof(i2s_sample_type) * AMY_NCHANS || read != AMY_BLOCK_SIZE * sizeof(i2s_sample_type) * AMY_NCHANS) {
             fprintf(stderr,"i2s underrun: [w %d,r %d] vs %d\n", written, read, AMY_BLOCK_SIZE * sizeof(i2s_sample_type) * AMY_NCHANS);
         }
-
+        if(audio_buffer_callback != NULL) {
+            mp_sched_schedule(audio_buffer_callback, mp_obj_new_int(osc));
+        }
     }
 }
 extern void esp_render_task( void * pvParameters);

tulip/amyboard/amychip.c

@@ -129,7 +129,7 @@ void setup_fs() {
 void mp_js_init(int pystack_size, int heap_size) {
 
     setup_fs();
-    emscripten_set_main_loop(main_loop__tulip, 60, 0);
+    emscripten_set_main_loop(main_loop__tulip, 0, 0);
 
     #if MICROPY_ENABLE_PYSTACK
     mp_obj_t *pystack = (mp_obj_t *)malloc(pystack_size * sizeof(mp_obj_t));

tulip/amyboardweb/main.c

@@ -50,6 +50,7 @@ enum {
     PROXY_KIND_MP_OBJECT = 8,
     PROXY_KIND_MP_JSPROXY = 9,
     PROXY_KIND_MP_EXISTING = 10,
+    PROXY_KIND_MP_BYTES = 11,
 };
 
 enum {
@@ -61,6 +62,7 @@ enum {
     PROXY_KIND_JS_STRING = 5,
     PROXY_KIND_JS_OBJECT = 6,
     PROXY_KIND_JS_PYPROXY = 7,
+    PROXY_KIND_JS_BYTES = 8,
 };
 
 MP_DEFINE_CONST_OBJ_TYPE(
@@ -170,6 +172,10 @@ mp_obj_t proxy_convert_js_to_mp_obj_cside(uint32_t *value) {
         mp_obj_t s = mp_obj_new_str((void *)value[2], value[1]);
         free((void *)value[2]);
         return s;
+    } else if (value[0] == PROXY_KIND_JS_BYTES) {
+        mp_obj_t s = mp_obj_new_bytes((void *) value[2], value[1]);
+        free((void *)value[2]);
+        return s;
     } else if (value[0] == PROXY_KIND_JS_PYPROXY) {
         return proxy_c_get_obj(value[1]);
     } else {
@@ -200,6 +206,14 @@ void proxy_convert_mp_to_js_obj_cside(mp_obj_t obj, uint32_t *out) {
         const char *str = mp_obj_str_get_data(obj, &len);
         out[1] = len;
         out[2] = (uintptr_t)str;
+    } else if (mp_obj_get_type(obj) == &mp_type_bytes || mp_obj_get_type(obj) == &mp_type_bytearray) {
+        kind = PROXY_KIND_MP_BYTES;
+        mp_buffer_info_t bufinfo;
+        mp_get_buffer(obj, &bufinfo, MP_BUFFER_READ);
+        size_t len = bufinfo.len;
+        const uint8_t *buf = bufinfo.buf;
+        out[1] = len;
+        out[2] = (uintptr_t)buf;
     } else if (obj == mp_const_undefined) {
         kind = PROXY_KIND_MP_JSPROXY;
         out[1] = 1;

tulip/amyboardweb/proxy_c.c

@@ -41,6 +41,7 @@ const PROXY_KIND_MP_GENERATOR = 7;
 const PROXY_KIND_MP_OBJECT = 8;
 const PROXY_KIND_MP_JSPROXY = 9;
 const PROXY_KIND_MP_EXISTING = 10;
+const PROXY_KIND_MP_BYTES = 11;
 
 const PROXY_KIND_JS_UNDEFINED = 0;
 const PROXY_KIND_JS_NULL = 1;
@@ -50,6 +51,8 @@ const PROXY_KIND_JS_DOUBLE = 4;
 const PROXY_KIND_JS_STRING = 5;
 const PROXY_KIND_JS_OBJECT = 6;
 const PROXY_KIND_JS_PYPROXY = 7;
+const PROXY_KIND_JS_BYTES = 8;
+
 
 class PythonError extends Error {
     constructor(exc_type, exc_details) {
@@ -196,7 +199,15 @@ function proxy_convert_js_to_mp_obj_jsside(js_obj, out) {
         Module.stringToUTF8(js_obj, buf, len + 1);
         Module.setValue(out + 4, len, "i32");
         Module.setValue(out + 8, buf, "i32");
-    } else if (
+    } else if (js_obj instanceof Uint8Array) {
+        kind = PROXY_KIND_JS_BYTES;
+        const len = js_obj.length;
+        const buf = Module._malloc(len);
+        for(let i = 0; i < len; i++) Module.HEAPU8[buf + i] = js_obj[i];
+        Module.setValue(out + 4, len, "i32");
+        Module.setValue(out + 8, buf, "i32");
+    } else if
+        (
         js_obj instanceof PyProxy ||
         (typeof js_obj === "function" && "_ref" in js_obj) ||
         js_obj instanceof PyProxyThenable
@@ -265,6 +276,11 @@ function proxy_convert_mp_to_js_obj_jsside(value) {
         const str_len = Module.getValue(value + 4, "i32");
         const str_ptr = Module.getValue(value + 8, "i32");
         obj = Module.UTF8ToString(str_ptr, str_len);
+    } else if (kind === PROXY_KIND_MP_BYTES) {
+        // bytes
+        const buf_len = Module.getValue(value + 4, "i32");
+        const buf_ptr = Module.getValue(value + 8, "i32");
+        obj = new Uint8Array(Module.HEAPU8.buffer, buf_ptr, buf_len);
     } else if (kind === PROXY_KIND_MP_JSPROXY) {
         // js proxy
         const id = Module.getValue(value + 4, "i32");

tulip/amyboardweb/proxy_js.js

@@ -72,5 +72,96 @@ js.fetch(url).then(lambda r: r.text()).then(lambda x: json.loads(x)).then(lambda
 amy.send(osc=30, pan=0, wave=amy.AUDIO_IN0, vel=1)
 amy.send(osc=31, pan=1, wave=amy.AUDIO_IN1, vel=1)
 amy.echo(level=1, delay_ms=400, max_delay_ms=1500, feedback=0.8, filter_coef=None)
-`}
-]
+`},{
+    't':'music',
+    'd':'Directly play a wav file',
+    'c':`
+# wav.py
+# plays a wavfile from amy external audio
+
+import amy_wave
+f = None
+try:
+    f = amy_wave.open(wav_filename,'rb')
+except:
+    print("Upload a wav file and type >>> wav_filename='file.wav' first.")
+
+def cb(x):
+    frames = f.readframes(256)
+    if(len(frames)!=1024):
+        frames = bytes(1024)
+        tulip.amy_block_done_callback()
+    tulip.amy_set_external_input_buffer(frames)
+
+if(f is not None):
+    amy.reset()
+    amy.send(osc=0,wave=amy.AUDIO_EXT0, pan=0, vel=1)
+    amy.send(osc=1,wave=amy.AUDIO_EXT1, pan=1, vel=1)
+    tulip.amy_block_done_callback(cb)
+`},{
+    't':'music',
+    'd':'Sample audio in and play it as an instrument',
+    'c':`
+# sample.py 
+# try sampling audio to pcm memorypcm
+
+import tulip, amy, synth, music
+
+buf = bytes()
+t = 0
+tick_start = 0
+ms = 2000
+syn = None
+
+def play():
+    amy.send(reset=amy.RESET_TIMEBASE+amy.RESET_ALL_OSCS)
+    amy.load_sample_bytes(buf, stereo=True, patch=50, loopstart=0, loopend=int(amy.AMY_SAMPLE_RATE*(ms/1000.0)))
+    syn = synth.OscSynth(wave=amy.PCM, patch=50)
+    for i,note in enumerate(music.Chord('F:min7').midinotes()):
+        syn.note_on(note+24, 1, time=i*1000)
+        syn.note_off(note+24, time=5000)
+
+def sample(x):
+    global buf, syn
+    buf = buf + tulip.amy_get_input_buffer()
+    if(tulip.ticks_ms() > tick_start + ms): 
+        tulip.amy_block_done_callback()
+        play()
+
+tick_start = tulip.ticks_ms()
+print("Recording for 2s. Make sure audio input is on!")
+tulip.amy_block_done_callback(sample)
+`
+},{
+    'd':'Generate audio buffers in Python',
+    't':'music',
+    'c':`
+# makes a sine wave. obviously, it's easier to do this in AMY directly! 
+# but just showing how to do it for any audio synthesis in python
+from ulab import numpy as np
+import ulab
+freq = 440.0
+amp = 32767
+lut_size = 512
+count = 0
+lut = np.array(amp * np.sin(np.linspace(0,2*np.pi,lut_size,endpoint=False)), dtype=np.int16)
+# freq/2 because we send stereo buffer in this example
+step_size = ((freq/2.0) * lut_size) / float(amy.AMY_SAMPLE_RATE)
+index = 0
+
+def cb(x):
+    global count
+    t = np.array(np.arange(count*512,count*512+512)*step_size)
+    samples = ulab.user.arraymodlut(lut, t);
+    tulip.amy_set_external_input_buffer(samples.tobytes())
+    if(count==1000):  # stop
+        tulip.amy_block_done_callback()
+        amy.reset()
+    count = count + 1
+
+amy.reset()
+amy.send(osc=0,wave=amy.AUDIO_EXT0, pan=0, vel=1)
+amy.send(osc=1,wave=amy.AUDIO_EXT1, pan=1, vel=1)
+tulip.amy_block_done_callback(cb)
+`
+}]