Switching to version 1.1 of Azure Kinect Body Tracking SDK, plus mino…

…r fixes and improvements

K4AdotNet.Samples.Core.BodyTrackingSpeed/DnnModel.cs

@@ -0,0 +1,8 @@
+namespace K4AdotNet.Samples.Core.BodyTrackingSpeed
+{
+    internal enum DnnModel
+    {
+        Default,
+        Lite,
+    }
+}

K4AdotNet.Samples.Core.BodyTrackingSpeed/ProcessingParameters.cs

@@ -1,6 +1,8 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.IO;
+using System.Linq;
 
 namespace K4AdotNet.Samples.Core.BodyTrackingSpeed
 {
@@ -9,7 +11,8 @@ internal sealed class ProcessingParameters
         public const string MKV_FILE_EXTENSION = ".mkv";
 
         public static readonly string MkvPathDescription = "Path to MKV file";
-        public static readonly string CpuOnlyModeDescription = "CPU/GPU mode (C - use only CPU, G - use GPU, default - G)";
+        public static readonly string ProcessingModeDescription = "Processing mode (C - CPU, G - GPU, U - CUDA, T - TensorRT, D - DirectML, default - C)";
+        public static readonly string DnnModelDescription = "DNN model (D - Default, L - Lite, default - D)";
         public static readonly string ImplementationDescription = "Optional implementation type (S - single thread, P - pop in background, E - enqueue in background, default - S)";
         public static readonly string StartTimeDescription = "Optional start time of video interval in seconds (default - beginning of recording)";
         public static readonly string EndTimeDescription = "Optional end time of video interval in seconds (default - end of recording)";
@@ -28,7 +31,8 @@ public static bool IsValueLikeToMkvFilePath(string value)
         }
 
         public string? MkvPath { get; private set; }
-        public bool CpuOnlyMode { get; private set; }
+        public BodyTracking.TrackerProcessingMode ProcessingMode { get; private set; }
+        public DnnModel DnnModel { get; private set; }
         public ProcessingImplementation Implementation { get; private set; }
         public TimeSpan? StartTime { get; private set; }
         public TimeSpan? EndTime { get; private set; }
@@ -71,32 +75,76 @@ public bool TrySetMkvPath(string value, [NotNullWhen(returnValue: false)] out st
             return true;
         }
 
-        public bool TrySetCpuOnlyMode(string value, [NotNullWhen(returnValue: false)] out string? message)
+        private static readonly IReadOnlyDictionary<string, BodyTracking.TrackerProcessingMode> processingModes
+            = new Dictionary<string, BodyTracking.TrackerProcessingMode>
+            {
+                ["c"] = BodyTracking.TrackerProcessingMode.Cpu,
+                ["g"] = BodyTracking.TrackerProcessingMode.Gpu,
+                ["u"] = BodyTracking.TrackerProcessingMode.GpuCuda,
+                ["t"] = BodyTracking.TrackerProcessingMode.GpuTensorRT,
+                ["d"] = BodyTracking.TrackerProcessingMode.GpuDirectML,
+            };
+
+        public bool TrySetProcessingMode(string value, [NotNullWhen(returnValue: false)] out string? message)
         {
             if (string.IsNullOrWhiteSpace(value))
             {
-                CpuOnlyMode = false;
+                ProcessingMode = BodyTracking.TrackerProcessingMode.Cpu;
                 message = null;
                 return true;
             }
 
             value = value.Trim().ToLowerInvariant();
-            switch (value)
+            if (processingModes.TryGetValue(value, out var mode))
             {
-                case "c":
-                    CpuOnlyMode = true;
-                    message = null;
-                    return true;
-                case "g":
-                    CpuOnlyMode = false;
-                    message = null;
-                    return true;
+                ProcessingMode = mode;
+                message = null;
+                return true;
             }
 
-            message = $"Invalid value. Expected 'C' or 'G'.";
+            message = InvalidValueMessage(processingModes.Keys);
             return false;
         }
 
+        private static string InvalidValueMessage(IEnumerable<string> possibleValues)
+            => "Invalid value. Expected " + string.Join(" or ", possibleValues.Select(s => "'" + s.ToUpperInvariant() + "'")) + " characters.";
+
+        private static readonly IReadOnlyDictionary<string, DnnModel> dnnModels
+            = new Dictionary<string, DnnModel>
+            {
+                ["d"] = DnnModel.Default,
+                ["l"] = DnnModel.Lite,
+            };
+
+        public bool TrySetDnnModel(string value, [NotNullWhen(returnValue: false)] out string? message)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                DnnModel = DnnModel.Default;
+                message = null;
+                return true;
+            }
+
+            value = value.Trim().ToLowerInvariant();
+            if (dnnModels.TryGetValue(value, out var model))
+            {
+                DnnModel = model;
+                message = null;
+                return true;
+            }
+
+            message = InvalidValueMessage(dnnModels.Keys);
+            return false;
+        }
+
+        private static readonly IReadOnlyDictionary<string, ProcessingImplementation> implementations
+            = new Dictionary<string, ProcessingImplementation>
+            {
+                ["s"] = ProcessingImplementation.SingleThread,
+                ["p"] = ProcessingImplementation.PopInBackground,
+                ["e"] = ProcessingImplementation.EnqueueInBackground,
+            };
+
         public bool TrySetImplementation(string value, [NotNullWhen(returnValue: false)] out string? message)
         {
             if (string.IsNullOrWhiteSpace(value))
@@ -107,23 +155,14 @@ public bool TrySetImplementation(string value, [NotNullWhen(returnValue: false)]
             }
 
             value = value.Trim().ToLowerInvariant();
-            switch (value)
-            {
-                case "s":
-                    Implementation = ProcessingImplementation.SingleThread;
-                    message = null;
-                    return true;
-                case "p":
-                    Implementation = ProcessingImplementation.PopInBackground;
-                    message = null;
-                    return true;
-                case "e":
-                    Implementation = ProcessingImplementation.EnqueueInBackground;
-                    message = null;
-                    return true;
-            }
-
-            message = $"Invalid value. Expected 'S' or 'P' or 'E' characters.";
+            if (implementations.TryGetValue(value, out var impl))
+            {
+                Implementation = impl;
+                message = null;
+                return true;
+            }
+
+            message = InvalidValueMessage(implementations.Keys);
             return false;
         }
 

K4AdotNet.Samples.Core.BodyTrackingSpeed/Processor.cs

@@ -28,12 +28,19 @@ protected Processor(ProcessingParameters processingParameters)
             if (processingParameters.StartTime.HasValue)
                 Seek(processingParameters.StartTime.Value);
             var config = BodyTracking.TrackerConfiguration.Default;
-            config.ProcessingMode = processingParameters.CpuOnlyMode
-                ? BodyTracking.TrackerProcessingMode.Cpu
-                : BodyTracking.TrackerProcessingMode.Gpu;
-            tracker = new BodyTracking.Tracker(ref calibration, config);
+            config.ProcessingMode = processingParameters.ProcessingMode;
+            config.ModelPath = GetModelPath(processingParameters.DnnModel);
+            tracker = new BodyTracking.Tracker(in calibration, config);
         }
 
+        private static string GetModelPath(DnnModel dnnModel)
+            => dnnModel switch
+            {
+                DnnModel.Default => Sdk.BODY_TRACKING_DNN_MODEL_FILE_NAME,
+                DnnModel.Lite => Sdk.BODY_TRACKING_DNN_MODEL_LITE_FILE_NAME,
+                _ => throw new NotSupportedException(),
+            };
+
         public virtual void Dispose()
         {
             tracker.Dispose();

K4AdotNet.Samples.Core.BodyTrackingSpeed/Program.cs

@@ -15,16 +15,6 @@ private static void Main(string[] args)
 
             Console.WriteLine("Body tracking speed test on prerecorded video from Kinect for Azure device");
 
-            Console.WriteLine();
-            Console.WriteLine("Initializing body tracking runtime...");
-            if (!Sdk.TryInitializeBodyTrackingRuntime(out var msg))
-            {
-                Console.WriteLine("Body tracking cannot be used!");
-                Console.WriteLine(msg);
-                return;
-            }
-            Console.WriteLine();
-
             var processingParameters = args.Length == 0
                 ? AskProcessingParameters()
                 : ParseCommandLineArguments(args);
@@ -35,6 +25,7 @@ private static void Main(string[] args)
                 return;
             }
 
+            Console.WriteLine();
             Process(processingParameters);
 
             if (args.Length == 0)
@@ -63,7 +54,9 @@ private static void PrintProcessingStatus(Processor processor)
             var parameters = new ProcessingParameters();
             if (!AskParameter(ProcessingParameters.MkvPathDescription, parameters.TrySetMkvPath))
                 return null;
-            if (!AskParameter(ProcessingParameters.CpuOnlyModeDescription, parameters.TrySetCpuOnlyMode))
+            if (!AskParameter(ProcessingParameters.ProcessingModeDescription, parameters.TrySetProcessingMode))
+                return null;
+            if (!AskParameter(ProcessingParameters.DnnModelDescription, parameters.TrySetDnnModel))
                 return null;
             if (!AskParameter(ProcessingParameters.ImplementationDescription, parameters.TrySetImplementation))
                 return null;
@@ -105,7 +98,12 @@ private static bool AskParameter(string prompt, ParameterSetter setter)
                 {
                     case "-m":
                     case "--mode":
-                        if (!ParseArgument(args, ref i, parameters.TrySetCpuOnlyMode))
+                        if (!ParseArgument(args, ref i, parameters.TrySetProcessingMode))
+                            return null;
+                        break;
+                    case "-d":
+                    case "--dnnModel":
+                        if (!ParseArgument(args, ref i, parameters.TrySetDnnModel))
                             return null;
                         break;
                     case "-i":
@@ -176,8 +174,9 @@ private static void PrintHowToUse()
             Console.WriteLine("where: ");
             Console.WriteLine("  <mkvFile> - " + ProcessingParameters.MkvPathDescription);
             Console.WriteLine("  options:");
-            Console.WriteLine("    -m, --mode c|g\t\t" + ProcessingParameters.StartTimeDescription);
-            Console.WriteLine("    -i, --implementation s|p|e\t\t" + ProcessingParameters.StartTimeDescription);
+            Console.WriteLine("    -m, --mode c|g|u|t|d\t\t" + ProcessingParameters.ProcessingModeDescription);
+            Console.WriteLine("    -d, --dnnMode d|l\t\t" + ProcessingParameters.DnnModelDescription);
+            Console.WriteLine("    -i, --implementation s|p|e\t\t" + ProcessingParameters.ImplementationDescription);
             Console.WriteLine("    -s, --startTime <time>\t\t" + ProcessingParameters.StartTimeDescription);
             Console.WriteLine("    -e, --endTime <time>\t\t" + ProcessingParameters.EndTimeDescription);
             Console.WriteLine();

K4AdotNet.Samples.Unity/Assets/Scripts/ApplicationController.cs

@@ -37,7 +37,7 @@ private IEnumerator Start()
                 _errorMessage.Show(
                     "Cannot initialize Azure Kinect Body Tracking runtime.\n" +
                     "Make sure that there are the following files in Assets\\Plugins\\K4AdotNet folder:\n" +
-                    "1. k4abt.dll\n2. dnn_model_2_0.onnx\n3. cudnn64_7.dll\n4. cublas64_100.dll\n5. cudart64_100.dll\n6. onnxruntime.dll\n7. vcomp140.dll");
+                    "1. k4abt.dll\n2. dnn_model_2_0_op11.onnx\n3. cublas64_11.dll\n4. cublasLt64_11.dll\n5. cudart64_110.dll\n6. cudnn_cnn_infer64_8.dll\n7. cudnn_ops_infer64_8.dll\n8. cudnn64_8.dll\n9. cufft64_10.dll\n10. onnxruntime.dll\n11. vcomp140.dll");
                 yield break;
             }
 

K4AdotNet.Samples.Unity/Assets/Scripts/SkeletonProvider.cs

@@ -21,7 +21,7 @@ private IEnumerator Start()
 
             var task = Task.Run(() =>
             {
-                var initialized = Sdk.TryInitializeBodyTrackingRuntime(out var message);
+                var initialized = Sdk.TryInitializeBodyTrackingRuntime(TrackerProcessingMode.GpuCuda, out var message);
                 return Tuple.Create(initialized, message);
             });
             yield return new WaitUntil(() => task.IsCompleted);
@@ -48,7 +48,13 @@ private IEnumerator Start()
                 if (captureManager?.IsAvailable == true)
                 {
                     var calibration = captureManager.Calibration;
-                    _tracker = new Tracker(ref calibration);
+
+                    var config = TrackerConfiguration.Default;
+                    config.ProcessingMode = TrackerProcessingMode.GpuCuda;
+                    // Use lite version of DNN model for speed (comment next line to use default DNN model)
+                    config.ModelPath = Sdk.BODY_TRACKING_DNN_MODEL_LITE_FILE_NAME;
+
+                    _tracker = new Tracker(ref calibration, config);
 
                     captureManager.CaptureReady += CaptureManager_CaptureReady;
                 }
@@ -75,7 +81,11 @@ private void CaptureManager_CaptureReady(object sender, CaptureEventArgs e)
         {
             if (IsAvailable)
             {
-                _tracker.TryEnqueueCapture(e.Capture);
+                using var capture = e.Capture;
+                using var depthImage = capture.DepthImage;
+                using var irImage = capture.IRImage;
+                if (!(depthImage is null) && !(irImage is null))
+                    _tracker.TryEnqueueCapture(capture);
             }
         }
 

K4AdotNet.Samples.Unity/ProjectSettings/ProjectVersion.txt

@@ -1,2 +1,2 @@
-m_EditorVersion: 2019.2.6f1
-m_EditorVersionWithRevision: 2019.2.6f1 (fe82a0e88406)
+m_EditorVersion: 2020.3.2f1
+m_EditorVersionWithRevision: 2020.3.2f1 (8fd9074bf66c)

K4AdotNet.Samples.Unity/prepare.cmd

@@ -13,13 +13,18 @@ xcopy ..\externals\k4a\windows-desktop\amd64\*.dll Assets\Plugins\K4AdotNet\ /A
 
 rem Copy k4abt.dll from K4A Body Tracking runtime to "Assets\Plugins\K4AdotNet" folder (assumes Body Tracking SDK installed into a standard location)
 copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\k4abt.dll" Assets\Plugins\K4AdotNet\
-copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\dnn_model_2_0.onnx" Assets\Plugins\K4AdotNet\
-copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cudnn64_7.dll" Assets\Plugins\K4AdotNet\
-copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cublas64_100.dll" Assets\Plugins\K4AdotNet\
-copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cudart64_100.dll" Assets\Plugins\K4AdotNet\
+copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\dnn_model_2_0_op11.onnx" Assets\Plugins\K4AdotNet\
+copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\dnn_model_2_0_lite_op11.onnx" Assets\Plugins\K4AdotNet\
+copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cublas64_11.dll" Assets\Plugins\K4AdotNet\
+copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cublasLt64_11.dll" Assets\Plugins\K4AdotNet\
+copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cudart64_110.dll" Assets\Plugins\K4AdotNet\
+copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cudnn_cnn_infer64_8.dll" Assets\Plugins\K4AdotNet\
+copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cudnn_ops_infer64_8.dll" Assets\Plugins\K4AdotNet\
+copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cudnn64_8.dll" Assets\Plugins\K4AdotNet\
+copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\cufft64_10.dll" Assets\Plugins\K4AdotNet\
 copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\onnxruntime.dll" Assets\Plugins\K4AdotNet\
 copy /y "%ProgramFiles%\Azure Kinect Body Tracking SDK\tools\vcomp140.dll" Assets\Plugins\K4AdotNet\
 
 rem If Body Tracking SDK is installed to another location or you want to have portable solution which does not require installation of Body Tracking SDK
 rem then copy manually the following libraries and data files from "tools" folder of Body Tracking SDK to "Assets\Plugins\K4AdotNet" folder of this plugin:
-rem k4abt.dll, dnn_model_2_0.onnx, cudnn64_7.dll, cublas64_100.dll, cudart64_100.dll, onnxruntime.dll, vcomp140.dll. 
+rem k4abt.dll, dnn_model_2_0_op11.onnx (or dnn_model_2_0_lite_op11.onnx), cublas64_11.dll, cublasLt64_11.dll, cudart64_110.dll, cudnn_cnn_infer64_8.dll, cudnn_ops_infer64_8.dll, cudnn64_8.dll, cufft64_10.dll, onnxruntime.dll, vcomp140.dll. 

K4AdotNet.Samples.Unity/readme.md

@@ -13,10 +13,14 @@ The script assumes that **Azure Kinect Body Tracking SDK** is installed into def
 If it doesn't take place or you want to have portable solution which doesn't depend on presence of Body Tracking SDK on local machine
 then copy the following files from `tools` folder of Body Tracking SDK to `Assets\Plugins\K4AdotNet` folder of this plugin:
 * `k4abt.dll`,
-* `dnn_model_2_0.onnx`,
-* `cudnn64_7.dll`,
-* `cublas64_100.dll`,
-* `cudart64_100.dll`,
+* `dnn_model_2_0_op11.onnx`,
+* `cublas64_11.dll`,
+* `cublasLt64_11.dll`,
+* `cudart64_110.dll`,
+* `cudnn_cnn_infer64_8.dll`,
+* `cudnn_ops_infer64_8.dll`,
+* `cudnn64_8.dll`,
+* `cufft64_10.dll`,
 * `onnxruntime.dll`,
 * `vcomp140.dll`.
 
@@ -32,10 +36,14 @@ This plugin depends on the following managed and native libraries and data files
   * `k4arecord.dll`
 * native libraries and ONNX-file from **Azure Kinect Body Tracking SDK 1.0.x** (you can download and install MSI package from [here](https://docs.microsoft.com/en-us/azure/kinect-dk/body-sdk-download) and find required files in `tools` subdirectory of installation destination directory):
   * `k4abt.dll`,
-  * `dnn_model_2_0.onnx`,
-  * `cudnn64_7.dll`,
-  * `cublas64_100.dll`,
-  * `cudart64_100.dll`,
+  * `dnn_model_2_0_op11.onnx`,
+  * `cublas64_11.dll`,
+  * `cublasLt64_11.dll`,
+  * `cudart64_110.dll`,
+  * `cudnn_cnn_infer64_8.dll`,
+  * `cudnn_ops_infer64_8.dll`,
+  * `cudnn64_8.dll`,
+  * `cufft64_10.dll`,
   * `onnxruntime.dll`,
   * `vcomp140.dll`.
 

K4AdotNet.Samples.Wpf.BodyTracker/BackgroundTrackingLoop.cs

@@ -11,21 +11,30 @@ internal sealed class BackgroundTrackingLoop : IDisposable
         private readonly Thread backgroundThread;
         private volatile bool isRunning;
 
-        public BackgroundTrackingLoop(ref Calibration calibration, bool cpuOnlyMode, SensorOrientation sensorOrientation, float smoothingFactor)
+        public BackgroundTrackingLoop(in Calibration calibration, TrackerProcessingMode processingMode, DnnModel dnnModel, SensorOrientation sensorOrientation, float smoothingFactor)
         {
             var config = new TrackerConfiguration
             {
                 SensorOrientation = sensorOrientation,
-                ProcessingMode = cpuOnlyMode
-                    ? TrackerProcessingMode.Cpu
-                    : TrackerProcessingMode.Gpu
+                ProcessingMode = processingMode,
+                ModelPath = GetModelPath(dnnModel),
             };
-            tracker = new Tracker(ref calibration, config) { TemporalSmoothingFactor = smoothingFactor };
+            tracker = new Tracker(in calibration, config) { TemporalSmoothingFactor = smoothingFactor };
             isRunning = true;
             backgroundThread = new Thread(BackgroundLoop) { IsBackground = true };
             backgroundThread.Start();
         }
 
+        private static string GetModelPath(DnnModel dnnModel)
+        {
+            switch (dnnModel)
+            {
+                case DnnModel.Default: return Sdk.BODY_TRACKING_DNN_MODEL_FILE_NAME;
+                case DnnModel.Lite: return Sdk.BODY_TRACKING_DNN_MODEL_LITE_FILE_NAME;
+                default: throw new NotSupportedException();
+            }
+    }
+
         public void Dispose()
         {
             if (isRunning)