Hey omi, using the device's programmable button (#1399)

Issue: #1388

## Feature
- "hey omi" should be triggered with a long press of the button on the
omi device and the query should be collected while the user is holding
the button.
 - turnon/turnoff of the device should be switched to short click
- after the button is released, we send the question/prompt to LLM,
while passing all {facts} and {conversations/memories}, chat history and
current conversation
 - 4 the answer should be shown in the chat and sent via notification

## Usages
- Long press to start the voice command (as a message for now), you will
feel the haptic feedback
- Release to send it, then wait for Omi's response.
- To turn the device on / off, use a single tap

## TODOs
- [x] "hey omi" should be triggered with a long press of the button on
the omi device and the query should be collected while the user is
holding the button.
 - [x] turnon/turnoff of the device should be switched to short click
- [x] after the button is released, we send the question/prompt to LLM,
while passing all {facts} and {conversations/memories}, chat history and
current conversation
- [x] 4 the answer should be shown in the chat and sent via notification
- [x] firmware: fix the button changes not notify sometime -> implement
the haptic then control it from the app + add new issue for firmware
fixes: #1412

## Deploy plan
- [ ] deploy backend
- [ ] deploy firmware
- [ ] deploy mobile app

Friend/firmware/firmware_v1.0/src/button.c

@@ -214,21 +214,23 @@ void check_button_level(struct k_work *work_item)
             {
                 //If button is pressed for a long time.......
                 notify_long_tap();
+                //play_haptic_milli(10);
                 //Fire the long mode notify and enter a grace period
                 //turn off herre
-                if(!from_wakeup)
-                {
-                    is_off = !is_off;
-                }
-                else
-                {
-                    from_wakeup = false;
-                }
-                if (is_off)
-                {
-                    bt_off();
-                    turnoff_all();
-                }
+                // TODO: FIXME
+                //if(!from_wakeup)
+                //{
+                //    is_off = !is_off;
+                //}
+                //else
+                //{
+                //    from_wakeup = false;
+                //}
+                //if (is_off)
+                //{
+                //    bt_off();
+                //    turnoff_all();
+                //}
                 current_button_state = GRACE;
                 reset_count();
             }
@@ -254,6 +256,20 @@ void check_button_level(struct k_work *work_item)
             else if (inc_count_0 > 10)
             {
                 notify_tap(); //Fire the notify and enter a grace period
+                if(!from_wakeup)
+                {
+                    is_off = !is_off;
+                }
+                else
+                {
+                    from_wakeup = false;
+                }
+                //Fire the notify and enter a grace period
+                if (is_off)
+                {
+                    bt_off();
+                    turnoff_all();
+                }
                 current_button_state = GRACE;
                 reset_count();
             }
@@ -272,20 +288,22 @@ void check_button_level(struct k_work *work_item)
             if (inc_count_1 > threshold) 
             {
                 notify_long_tap();
-                if(!from_wakeup)
-                {
-                    is_off = !is_off;
-                }
-                else
-                {
-                    from_wakeup = false;
-                }
-                //Fire the notify and enter a grace period
-                if (is_off)
-                {
-                    bt_off();
-                    turnoff_all();
-                }
+                //play_haptic_milli(10);
+                // TODO: FIXME
+                //if(!from_wakeup)
+                //{
+                //    is_off = !is_off;
+                //}
+                //else
+                //{
+                //    from_wakeup = false;
+                //}
+                ////Fire the notify and enter a grace period
+                //if (is_off)
+                //{
+                //    bt_off();
+                //    turnoff_all();
+                //}
                 current_button_state = GRACE;
                 reset_count();
             }
@@ -422,4 +440,4 @@ void turnoff_all()
 void force_button_state(FSM_STATE_T state)
 {
     current_button_state = state;
-}
+}

Friend/firmware/firmware_v1.0/src/speaker.c

@@ -1,4 +1,8 @@
 #include <zephyr/kernel.h>
+#include <zephyr/bluetooth/uuid.h>
+#include <zephyr/bluetooth/gatt.h>
+#include <zephyr/bluetooth/l2cap.h>
+#include <zephyr/bluetooth/services/bas.h>
 #include <zephyr/sys/printk.h>
 #include <zephyr/device.h>
 #include <zephyr/drivers/i2s.h>
@@ -39,6 +43,72 @@ struct gpio_dt_spec haptic_gpio_pin = {.port = DEVICE_DT_GET(DT_NODELABEL(gpio1)
 
 struct gpio_dt_spec speaker_gpio_pin = {.port = DEVICE_DT_GET(DT_NODELABEL(gpio0)), .pin=4, .dt_flags = GPIO_INT_DISABLE};
 
+// ble service
+//
+
+static void speaker_ccc_config_changed_handler(const struct bt_gatt_attr *attr, uint16_t value);
+static ssize_t speaker_haptic_handler(struct bt_conn *conn, const struct bt_gatt_attr *attr, const void *buf, uint16_t len, uint16_t offset, uint8_t flags);
+
+static struct bt_uuid_128 speaker_uuid = BT_UUID_INIT_128(BT_UUID_128_ENCODE(0xCAB1AB95, 0x2EA5, 0x4F4D, 0xBB56, 0x874B72CFC984));
+static struct bt_uuid_128 speaker_haptic_uuid = BT_UUID_INIT_128(BT_UUID_128_ENCODE(0xCAB1AB96, 0x2EA5, 0x4F4D, 0xBB56, 0x874B72CFC984));
+
+static struct bt_gatt_attr speaker_service_attr[] = {
+    BT_GATT_PRIMARY_SERVICE(&speaker_uuid),
+    BT_GATT_CHARACTERISTIC(&speaker_haptic_uuid.uuid, BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY, BT_GATT_PERM_WRITE, NULL, speaker_haptic_handler, NULL),
+    BT_GATT_CCC(speaker_ccc_config_changed_handler, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE),
+};
+static struct bt_gatt_service speaker_service = BT_GATT_SERVICE(speaker_service_attr);
+
+void register_speaker_service() 
+{
+    bt_gatt_service_register(&speaker_service);
+}
+
+static void speaker_ccc_config_changed_handler(const struct bt_gatt_attr *attr, uint16_t value) 
+{
+    if (value == BT_GATT_CCC_NOTIFY)
+    {
+        LOG_INF("Client subscribed for notifications");
+    }
+    else if (value == 0)
+    {
+        LOG_INF("Client unsubscribed from notifications");
+    }
+    else
+    {
+        LOG_ERR("Invalid CCC value: %u", value);
+    }
+
+}
+
+static ssize_t speaker_haptic_handler(struct bt_conn *conn, const struct bt_gatt_attr *attr, const void *buf, uint16_t len, uint16_t offset, uint8_t flags) 
+{
+    LOG_INF("play the haptic");
+
+    uint8_t value = ((uint8_t*)buf)[0];
+    LOG_INF("value %d  ", value);
+
+	if (value < 1 || value > 3)
+	{
+		return 0;
+	}
+
+	if (value == 1)
+	{
+		play_haptic_milli(20);
+	}
+	else if (value == 2)
+	{
+		play_haptic_milli(50);
+	}
+	else if (value == 3)
+	{
+		play_haptic_milli(500);
+	}
+
+    return 1;
+}
+
 int speaker_init() 
 {
     LOG_INF("Speaker init");

Friend/firmware/firmware_v1.0/src/speaker.h

@@ -53,4 +53,6 @@ void play_haptic_milli(uint32_t duration);
 
 void speaker_off();
 
-#endif
+void register_speaker_service();
+
+#endif

Friend/firmware/firmware_v1.0/src/storage.c

@@ -380,4 +380,4 @@ int storage_init()
 {
     k_thread_create(&storage_thread, storage_stack, K_THREAD_STACK_SIZEOF(storage_stack), (k_thread_entry_t)storage_write, NULL, NULL, NULL, K_PRIO_PREEMPT(7), 0, K_NO_WAIT);
     return 0;
-}
+}

Friend/firmware/firmware_v1.0/src/transport.c

@@ -799,10 +799,8 @@ int transport_start()
         LOG_ERR("Speaker failed to start");
         return 0;
     }
-    else
-    {
-        LOG_INF("Speaker initialized");
-    }
+    LOG_INF("Speaker initialized");
+    register_speaker_service();
 
 
 #endif
@@ -862,4 +860,4 @@ int broadcast_audio_packets(uint8_t *buffer, size_t size)
 void accel_off()
 {
     gpio_pin_set_dt(&accel_gpio_pin, 0);
-}
+}

app/lib/backend/http/api/messages.dart

@@ -1,11 +1,14 @@
 import 'dart:convert';
+import 'dart:io';
 
 import 'package:flutter/material.dart';
+import 'package:http/http.dart' as http;
 import 'package:friend_private/backend/http/shared.dart';
 import 'package:friend_private/backend/schema/message.dart';
 import 'package:friend_private/env/env.dart';
 import 'package:friend_private/utils/logger.dart';
 import 'package:instabug_flutter/instabug_flutter.dart';
+import 'package:path/path.dart';
 
 Future<List<ServerMessage>> getMessagesServer() async {
   // TODO: Add pagination
@@ -72,3 +75,29 @@ Future<ServerMessage> getInitialAppMessage(String? appId) {
     }
   });
 }
+
+Future<List<ServerMessage>> sendVoiceMessageServer(List<File> files) async {
+  var request = http.MultipartRequest(
+    'POST',
+    Uri.parse('${Env.apiBaseUrl}v1/voice-messages'),
+  );
+  for (var file in files) {
+    request.files.add(await http.MultipartFile.fromPath('files', file.path, filename: basename(file.path)));
+  }
+  request.headers.addAll({'Authorization': await getAuthHeader()});
+
+  try {
+    var streamedResponse = await request.send();
+    var response = await http.Response.fromStream(streamedResponse);
+    if (response.statusCode == 200) {
+      debugPrint('sendVoiceMessageServer response body: ${jsonDecode(response.body)}');
+      return ((jsonDecode(response.body) ?? []) as List<dynamic>).map((m) => ServerMessage.fromJson(m)).toList();
+    } else {
+      debugPrint('Failed to upload sample. Status code: ${response.statusCode} ${response.body}');
+      throw Exception('Failed to upload sample. Status code: ${response.statusCode}');
+    }
+  } catch (e) {
+    debugPrint('An error occurred uploadSample: $e');
+    throw Exception('An error occurred uploadSample: $e');
+  }
+}