[ML4SE-137] added EmojiConfig

ijPlugin/src/main/kotlin/org/jetbrains/research/tasktracker/config/DefaultConfigsFactory.kt

@@ -5,6 +5,7 @@ import org.jetbrains.research.tasktracker.config.content.FinalPageContentConfig
 import org.jetbrains.research.tasktracker.config.content.MainPageContentConfig
 import org.jetbrains.research.tasktracker.config.content.ServerErrorPageConfig
 import org.jetbrains.research.tasktracker.config.content.TaskContentConfig
+import org.jetbrains.research.tasktracker.config.emoji.EmojiConfig
 import org.jetbrains.research.tasktracker.config.ide.inspection.InspectionConfig
 import org.jetbrains.research.tasktracker.config.ide.settings.SettingsConfig
 import org.jetbrains.research.tasktracker.config.scenario.ScenarioConfig
@@ -30,6 +31,7 @@ object DefaultConfigsFactory {
         FinalPageContentConfig.CONFIG_FILE_PREFIX,
         ServerErrorPageConfig.CONFIG_FILE_PREFIX,
         SurveyConfig.CONFIG_FILE_PREFIX,
+        EmojiConfig.CONFIG_FILE_PREFIX
     )
 
     fun createDefaultConfigs() {

ijPlugin/src/main/kotlin/org/jetbrains/research/tasktracker/config/MainTaskTrackerConfig.kt

@@ -6,6 +6,7 @@ import org.jetbrains.research.tasktracker.config.content.FinalPageContentConfig
 import org.jetbrains.research.tasktracker.config.content.MainPageContentConfig
 import org.jetbrains.research.tasktracker.config.content.ServerErrorPageConfig
 import org.jetbrains.research.tasktracker.config.content.TaskContentConfig
+import org.jetbrains.research.tasktracker.config.emoji.EmojiConfig
 import org.jetbrains.research.tasktracker.config.ide.MainIdeConfig
 import org.jetbrains.research.tasktracker.config.scenario.ScenarioConfig
 import org.jetbrains.research.tasktracker.config.survey.SurveyConfig
@@ -30,6 +31,7 @@ data class MainTaskTrackerConfig(
     var mainPageConfig: MainPageContentConfig? = null,
     var finalPageConfig: FinalPageContentConfig? = null,
     var serverErrorPageConfig: ServerErrorPageConfig? = null,
+    var emojiConfig: EmojiConfig? = null,
 
     var scenarioConfig: ScenarioConfig? = null,
     var surveyConfig: SurveyConfig? = null,
@@ -43,7 +45,8 @@ data class MainTaskTrackerConfig(
         webCamConfig,
         mainPageConfig,
         finalPageConfig,
-        serverErrorPageConfig
+        serverErrorPageConfig,
+        emojiConfig
     )
 
     companion object {
@@ -118,6 +121,12 @@ data class MainTaskTrackerConfig(
                             mainConfig.surveyConfig, configFile, SurveyConfig::buildConfig, logger
                         )
                     }
+
+                    fileName.startsWith(EmojiConfig.CONFIG_FILE_PREFIX) -> {
+                        mainConfig.emojiConfig = buildBaseConfig(
+                            mainConfig.emojiConfig, configFile, EmojiConfig::buildConfig, logger
+                        )
+                    }
                 }
             }
             mainConfig.mainIdeConfig = MainIdeConfig.buildConfig(configFiles)

ijPlugin/src/main/kotlin/org/jetbrains/research/tasktracker/config/emoji/Emoji.kt

@@ -0,0 +1,29 @@
+package org.jetbrains.research.tasktracker.config.emoji
+
+import com.intellij.openapi.util.IconLoader
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.Transient
+import javax.swing.Icon
+
+@Serializable
+data class Emoji(
+    val modelPosition: Int,
+    val name: String,
+    private val affirmDescriptions: List<String>,
+    private val adviceDescriptions: List<String> = emptyList()
+) {
+    private val iconName: String? = null
+    private val modalWindowIconName: String? = null
+
+    @Transient
+    val icon: Icon? = iconName?.let { IconLoader.getIcon(it, Emoji::class.java) }
+
+    @Transient
+    val modalWindowIcon: Icon? = modalWindowIconName?.let { IconLoader.getIcon(it, Emoji::class.java) }
+
+    val randomAffirmDescription
+        get() = affirmDescriptions.random()
+
+    val randomAdviceDescription
+        get() = adviceDescriptions.random()
+}

ijPlugin/src/main/kotlin/org/jetbrains/research/tasktracker/config/emoji/EmojiConfig.kt

@@ -0,0 +1,25 @@
+package org.jetbrains.research.tasktracker.config.emoji
+
+import kotlinx.serialization.Serializable
+import org.jetbrains.research.tasktracker.config.BaseConfig
+import org.jetbrains.research.tasktracker.config.YamlConfigLoadStrategy
+import org.jetbrains.research.tasktracker.handler.BaseHandler
+import org.jetbrains.research.tasktracker.handler.emoji.EmojiHandler
+import java.io.File
+
+@Serializable
+data class EmojiConfig(
+    val emojis: List<Emoji>
+) : BaseConfig {
+    override val configName: String
+        get() = "emoji"
+
+    override fun buildHandler(): BaseHandler = EmojiHandler(this)
+
+    companion object {
+        const val CONFIG_FILE_PREFIX: String = "emoji"
+
+        fun buildConfig(configFile: File): EmojiConfig =
+            YamlConfigLoadStrategy.load(configFile.readText(), serializer())
+    }
+}

ijPlugin/src/main/kotlin/org/jetbrains/research/tasktracker/handler/emoji/EmojiHandler.kt

@@ -0,0 +1,12 @@
+package org.jetbrains.research.tasktracker.handler.emoji
+
+import com.intellij.openapi.project.Project
+import org.jetbrains.research.tasktracker.config.BaseConfig
+import org.jetbrains.research.tasktracker.handler.BaseHandler
+
+class EmojiHandler(override val config: BaseConfig) : BaseHandler {
+
+    override fun setup(project: Project) {
+//        TODO("Setup emojis according to the config")
+    }
+}

ijPlugin/src/main/kotlin/org/jetbrains/research/tasktracker/modelInference/EmoClient.kt

@@ -1,55 +1,55 @@
-
-import io.ktor.client.*
-import io.ktor.client.call.*
-import io.ktor.client.engine.cio.*
-import io.ktor.client.plugins.contentnegotiation.*
-import io.ktor.client.request.*
-import io.ktor.http.*
-import io.ktor.serialization.kotlinx.json.*
-import kotlinx.serialization.Serializable
-import kotlinx.serialization.json.Json
-import org.jetbrains.research.tasktracker.modelInference.EmoPrediction
-import org.jetbrains.research.tasktracker.modelInference.EmoPredictor
-import org.jetbrains.research.tasktracker.modelInference.prepareImage
-import org.opencv.core.Mat
-
-@Serializable
-data class ImageData(val image: List<List<Double>>)
-class EmoClient(private var serverUrl: String = "http://localhost:5230/predict") : EmoPredictor {
-
-    private val client = HttpClient(CIO) {
-        install(ContentNegotiation) {
-            json(
-                Json {
-                    prettyPrint = true
-                    isLenient = true
-                },
-            )
-        }
-    }
-
-    override suspend fun predict(image: Mat): EmoPrediction {
-        val prepImage = prepareImage(image)
-
-        val imageList = mutableListOf<List<Double>>()
-        for (row in 0 until prepImage.rows()) {
-            val rowList = mutableListOf<Double>()
-            for (col in 0 until prepImage.cols()) {
-                val pixelValue = prepImage.get(row, col)[0]
-                rowList.add(pixelValue)
-            }
-            imageList.add(rowList)
-        }
-
-        val imageData = ImageData(imageList)
-
-        val response: Map<Int, Double> = client.post(serverUrl) {
-            contentType(ContentType.Application.Json)
-            setBody(imageData)
-        }.body()
-
-        println("Emotion Probabilities: $response")
-
-        return EmoPrediction(response)
-    }
-}
+//
+//import io.ktor.client.*
+//import io.ktor.client.call.*
+//import io.ktor.client.engine.cio.*
+//import io.ktor.client.plugins.contentnegotiation.*
+//import io.ktor.client.request.*
+//import io.ktor.http.*
+//import io.ktor.serialization.kotlinx.json.*
+//import kotlinx.serialization.Serializable
+//import kotlinx.serialization.json.Json
+//import org.jetbrains.research.tasktracker.modelInference.EmoPrediction
+//import org.jetbrains.research.tasktracker.modelInference.EmoPredictor
+//import org.jetbrains.research.tasktracker.modelInference.prepare
+//import org.opencv.core.Mat
+//
+//@Serializable
+//data class ImageData(val image: List<List<Double>>)
+//class EmoClient(private var serverUrl: String = "http://localhost:5230/predict") : EmoPredictor {
+//
+//    private val client = HttpClient(CIO) {
+//        install(ContentNegotiation) {
+//            json(
+//                Json {
+//                    prettyPrint = true
+//                    isLenient = true
+//                },
+//            )
+//        }
+//    }
+//
+//    override suspend fun predict(image: Mat): EmoPrediction {
+//        val prepImage = image.prepare()
+//
+//        val imageList = mutableListOf<List<Double>>()
+//        for (row in 0 until prepImage.rows()) {
+//            val rowList = mutableListOf<Double>()
+//            for (col in 0 until prepImage.cols()) {
+//                val pixelValue = prepImage.get(row, col)[0]
+//                rowList.add(pixelValue)
+//            }
+//            imageList.add(rowList)
+//        }
+//
+//        val imageData = ImageData(imageList)
+//
+//        val response: Map<Int, Double> = client.post(serverUrl) {
+//            contentType(ContentType.Application.Json)
+//            setBody(imageData)
+//        }.body()
+//
+//        println("Emotion Probabilities: $response")
+//
+//        return EmoPrediction(response)
+//    }
+//}

ijPlugin/src/main/kotlin/org/jetbrains/research/tasktracker/modelInference/EmoPredictor.kt

@@ -1,22 +1,27 @@
 package org.jetbrains.research.tasktracker.modelInference
 
+import com.intellij.openapi.diagnostic.Logger
 import org.opencv.core.Mat
 
 class EmoPrediction(val probabilities: Map<Int, Double>) {
+    private val logger = Logger.getInstance(EmoPrediction::class.java)
 
-    companion object {
-        private const val THRESHOLD = 0.1
-        private val SENSITIVE_RANGE: List<Int> = (7 downTo 2).toList()
-    }
     fun getPrediction(): Int {
         for (i in SENSITIVE_RANGE) {
-            if (probabilities[i]!! >= THRESHOLD) {
-                return i
-            }
+            probabilities[i]?.let {
+                if (it >= THRESHOLD) {
+                    return i
+                }
+            } ?: logger.warn("There are no probability by key `$i`")
         }
 
         return probabilities.maxBy { it.value }.key
     }
+
+    companion object {
+        private const val THRESHOLD = 0.1
+        private val SENSITIVE_RANGE: List<Int> = (7 downTo 2).toList()
+    }
 }
 
 interface EmoPredictor {

ijPlugin/src/main/kotlin/org/jetbrains/research/tasktracker/modelInference/ImageUtils.kt

@@ -11,16 +11,14 @@ fun resizeImage(image: Mat, pixels: Double = 64.0): Mat {
     return resizedImage
 }
 
-fun prepareImage(image: Mat): Mat {
-    val gImage = grayImage(image)
-    val resImage = resizeImage(gImage)
-
-    return resImage
+fun Mat.prepare(): Mat {
+    val gImage = this.toGrayImage()
+    return resizeImage(gImage)
 }
 
-fun grayImage(image: Mat): Mat {
+fun Mat.toGrayImage(): Mat {
     val grayImage = Mat()
-    Imgproc.cvtColor(image, grayImage, Imgproc.COLOR_RGB2GRAY)
+    Imgproc.cvtColor(this, grayImage, Imgproc.COLOR_RGB2GRAY)
 
     return grayImage
 }

ijPlugin/src/main/kotlin/org/jetbrains/research/tasktracker/modelInference/model/EmoModel.kt

@@ -9,64 +9,44 @@ import kotlinx.coroutines.runBlocking
 import org.jetbrains.research.tasktracker.modelInference.EmoPrediction
 import org.jetbrains.research.tasktracker.modelInference.EmoPredictor
 import org.jetbrains.research.tasktracker.modelInference.getPixel
-import org.jetbrains.research.tasktracker.modelInference.prepareImage
+import org.jetbrains.research.tasktracker.modelInference.prepare
 import org.opencv.core.Mat
-import org.opencv.imgcodecs.Imgcodecs
-import kotlin.math.exp
 
 class EmoModel : EmoPredictor {
 
-    private fun softmax(input: FloatArray): FloatArray {
-        val max = (input.maxOrNull() ?: 0.0) as Float
-        val expSum = input.map { exp(it - max) }.sum()
-
-        return input.map { exp(it - max) / expSum }.toFloatArray()
-    }
-
-    companion object {
-        private const val MODEL_PATH = "emotion-ferplus-18.onnx"
-        private val INPUT_SHAPE = intArrayOf(1, 1, 64, 64)
-    }
-
     init {
         runBlocking {
-            load()
+            loadModel()
         }
     }
 
     private lateinit var model: KIModel
-    suspend fun load() {
-//        model = KIEngine.loadModel(EmoModel::class.java.getResource(MODEL_PATH).readBytes())
+    private suspend fun loadModel() {
         model = KIEngine.loadModel(
             EmoModel::class.java
                 .getResource(MODEL_PATH)?.readBytes() ?: error("$MODEL_PATH must exist")
         )
     }
 
     override suspend fun predict(image: Mat): EmoPrediction {
-        val prepImage = prepareImage(image)
+        val prepImage = image.prepare()
         val tensor = FloatNDArray(INPUT_SHAPE) { idx: IntArray ->
             getPixel(idx, prepImage)
         }
-
+        // TODO Rewrite to constants
         val outputs = model.predict(listOf(tensor.asTensor("Input3")))
         val output = outputs["Plus692_Output_0"]
-        val softmaxedOutput = ((output as KITensor).data as FloatNDArray).softmax()
-        val outputArray = softmaxedOutput.array.toArray()
+        val softmaxOutput = ((output as KITensor).data as FloatNDArray).softmax()
+        val outputArray = softmaxOutput.array.toArray()
 
         val probabilities = outputArray.mapIndexed { index: Int, prob: Float -> index to prob.toDouble() }.toMap()
-        println(probabilities)
         return EmoPrediction(probabilities)
     }
-}
 
-fun main() {
-    val dir = "/Users/maria.tigina/IdeaProjects/emotional-monitoring/ijPlugin/src/main/resources/img/"
-    val inputImage: Mat = Imgcodecs.imread(dir + "img.png")
 
-    runBlocking {
-        val model = EmoModel()
-        val results = model.predict(inputImage)
-        println(results)
+    companion object {
+        private const val MODEL_PATH = "emotion-ferplus-18.onnx"
+        private val INPUT_SHAPE = intArrayOf(1, 1, 64, 64)
     }
+
 }